def test_init_invalid_options_should_raise(self):
"""
Test __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts)
"""
scan_id = str(uuid.uuid4())
invalid_types = [None, "", list()]
for invalid_type in invalid_types:
with self.subTest(invalid_type=invalid_type):
with self.assertRaises(TypeError) as cm:
sfscan = SpiderFootScanner("",
scan_id,
"spiderfoot.net",
"IP_ADDRESS",
list(),
invalid_type,
start=False)
scan_id = str(uuid.uuid4())
with self.assertRaises(ValueError) as cm:
sfscan = SpiderFootScanner("",
scan_id,
"",
"IP_ADDRESS",
list(),
dict(),
start=False)
def test_set_status_invalid_status_should_raise(self):
"""
Test def setStatus(self, status, started=None, ended=None)
"""
opts = self.default_options
opts['__modules__'] = dict()
scan_id = str(uuid.uuid4())
sfscan = SpiderFootScanner("", scan_id, "spiderfoot.net", "IP_ADDRESS", list(), opts, start=False)
self.assertIsInstance(sfscan, SpiderFootScanner)
with self.assertRaises(ValueError) as cm:
sfscan.setStatus("invalid status", None, None)
def test__setStatus_argument_status_with_blank_value_should_raise_ValueError(self):
"""
Test __setStatus(self, status, started=None, ended=None)
"""
opts = self.default_options
opts['__modules__'] = dict()
scan_id = str(uuid.uuid4())
module_list = ['sfp__stor_db']
sfscan = SpiderFootScanner("example scan name", scan_id, "spiderfoot.net", "IP_ADDRESS", module_list, opts, start=False)
with self.assertRaises(ValueError):
sfscan._SpiderFootScanner__setStatus("example invalid scan status")
def test_get_id_should_return_a_scan_id(self):
"""
Test def getId(self)
"""
opts = self.default_options
opts['__modules__'] = dict()
scan_id = str(uuid.uuid4())
sfscan = SpiderFootScanner("", scan_id, "spiderfoot.net", "IP_ADDRESS", list(), opts, start=False)
self.assertIsInstance(sfscan, SpiderFootScanner)
get_id = sfscan.getId()
self.assertIsInstance(get_id, str)
self.assertEqual(scan_id, get_id)
def test_init_argument_targetType_invalid_value_should_raise_ValueError(self):
"""
Test __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts)
"""
scan_id = str(uuid.uuid4())
module_list = ['sfp__stor_db']
target_type = ""
with self.assertRaises(ValueError):
SpiderFootScanner("example scan name", scan_id, "spiderfoot.net", target_type, module_list, self.default_options, start=False)
target_type = "INVALID_TARGET_TYPE"
with self.assertRaises(ValueError):
SpiderFootScanner("example scan name", scan_id, "spiderfoot.net", target_type, module_list, self.default_options, start=False)
def test__setStatus_argument_status_of_invalid_type_should_raise_TypeError(self):
"""
Test __setStatus(self, status, started=None, ended=None)
"""
opts = self.default_options
opts['__modules__'] = dict()
scan_id = str(uuid.uuid4())
module_list = ['sfp__stor_db']
sfscan = SpiderFootScanner("example scan name", scan_id, "spiderfoot.net", "IP_ADDRESS", module_list, opts, start=False)
invalid_types = [None, list(), dict(), int()]
for invalid_type in invalid_types:
with self.subTest(invalid_type=invalid_type):
with self.assertRaises(TypeError):
sfscan._SpiderFootScanner__setStatus(invalid_type)
def test_init_blank_targettype_value_should_raise(self):
"""
Test __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts)
"""
scan_id = str(uuid.uuid4())
with self.assertRaises(ValueError) as cm:
sfscan = SpiderFootScanner("", scan_id, "spiderfoot.net", "invalid target type", list(), self.default_options, start=False)
def test_init_no_options_should_raise(self):
"""
Test __init__(self, scanName, scanTarget, targetType, scanId, moduleList, globalOpts, moduleOpts)
"""
with self.assertRaises(TypeError) as cm:
sfscan = SpiderFootScanner(None, None, None, None, None, None,
None)
def test_init_blank_targetvalue_value_should_raise(self):
"""
Test __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts, start=True)
"""
scan_id = str(uuid.uuid4())
with self.assertRaises(TypeError) as cm:
sfscan = SpiderFootScanner("", scan_id, None, "IP_ADDRESS", list(), self.default_options, start=False)
def rerunscan(self, id):
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
info = dbh.scanInstanceGet(id)
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if len(scanconfig) == 0:
return self.error("Something went wrong internally.")
modlist = scanconfig['_modulesenabled'].split(',')
if "sfp__stor_stdout" in modlist:
modlist.remove("sfp__stor_stdout")
targetType = sf.targetType(scantarget)
if targetType == None:
# It must then be a name, as a re-run scan should always have a clean
# target.
targetType = "HUMAN_NAME"
if targetType != "HUMAN_NAME":
scantarget = scantarget.lower()
# Start running a new scan
newId = sf.genScanInstanceGUID(scanname)
t = SpiderFootScanner(scanname, scantarget, targetType, newId, modlist,
cfg, modopts)
t.start()
# Wait until the scan has initialized
while globalScanStatus.getStatus(newId) == None:
print("[info] Waiting for the scan to initialize...")
time.sleep(1)
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
return templ.render(id=newId,
name=str(scanname),
docroot=self.docroot,
status=globalScanStatus.getStatus(newId),
pageid="SCANLIST")
def test_init_argument_start_true_with_no_valid_modules_should_set_scanstatus_to_failed(self):
opts = self.default_options
opts['__modules__'] = dict()
scan_id = str(uuid.uuid4())
module_list = ['invalid module']
sfscan = SpiderFootScanner("example scan name", scan_id, "spiderfoot.net", "INTERNET_NAME", module_list, opts, start=True)
self.assertIsInstance(sfscan, SpiderFootScanner)
self.assertEqual(sfscan.status, "ERROR-FAILED")
def test_init_argument_scanId_as_empty_string_should_raise_ValueError(self):
"""
Test __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts, start=True)
"""
scan_id = ""
module_list = ['sfp__stor_db']
with self.assertRaises(ValueError):
SpiderFootScanner("example scan name", scan_id, "spiderfoot.net", "IP_ADDRESS", module_list, self.default_options, start=False)
def test_init_argument_start_true_should_create_and_start_a_scan(self):
opts = self.default_options
opts['__modules__'] = dict()
scan_id = str(uuid.uuid4())
module_list = ['sfp__stor_db']
sfscan = SpiderFootScanner("example scan name", scan_id, "spiderfoot.net", "IP_ADDRESS", module_list, opts, start=True)
self.assertIsInstance(sfscan, SpiderFootScanner)
self.assertEqual(sfscan.status, "FINISHED")
def test_init_argument_globalOpts_as_empty_dict_should_raise_ValueError(self):
"""
Test __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts)
"""
scan_id = str(uuid.uuid4())
module_list = ['sfp__stor_db']
with self.assertRaises(ValueError):
SpiderFootScanner("example scan name", scan_id, "spiderfoot.net", "IP_ADDRESS", module_list, dict(), start=False)
def test_init(self):
"""
Test __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts, start=True)
"""
opts = self.default_options
opts['__modules__'] = dict()
scan_id = str(uuid.uuid4())
sfscan = SpiderFootScanner("", scan_id, "spiderfoot.net", "IP_ADDRESS", list(), opts, start=False)
self.assertIsInstance(sfscan, SpiderFootScanner)
def startscan(self, scanname, scantarget, modulelist, typelist):
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(self.config)
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
[scanname, scantarget] = self.cleanUserInput([scanname, scantarget])
if scanname == "" or scantarget == "":
return self.error("Form incomplete.")
if typelist == "" and modulelist == "":
return self.error("Form incomplete.")
if modulelist != "":
modlist = modulelist.replace('module_', '').split(',')
else:
typesx = typelist.replace('type_', '').split(',')
# 1. Find all modules that produce the requested types
modlist = sf.modulesProducing(typesx)
newmods = deepcopy(modlist)
newmodcpy = deepcopy(newmods)
# 2. For each type those modules consume, get modules producing
while len(newmodcpy) > 0:
for etype in sf.eventsToModules(newmodcpy):
xmods = sf.modulesProducing([etype])
for mod in xmods:
if mod not in modlist:
modlist.append(mod)
newmods.append(mod)
newmodcpy = deepcopy(newmods)
newmods = list()
# Add our mandatory storage module..
if "sfp__stor_db" not in modlist:
modlist.append("sfp__stor_db")
modlist.sort()
# For now we don't permit multiple simultaneous scans
for thread in threading.enumerate():
if thread.name.startswith("SF_"):
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(modules=self.config['__modules__'],
alreadyRunning=True, runningScan=thread.name[3:],
types=types, pageid="NEWSCAN")
# Start running a new scan
self.scanner = SpiderFootScanner(scanname, scantarget.lower(), modlist,
self.config, modopts)
t = threading.Thread(name="SF_" + scanname, target=self.scanner.startScan)
t.start()
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
return templ.render(id=self.scanner.myId, name=scanname,
status=self.scanner.status, pageid="SCANLIST")
def rerunscanmulti(self, ids):
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
for id in ids.split(","):
info = dbh.scanInstanceGet(id)
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if len(scanconfig) == 0:
return self.error("Something went wrong internally.")
modlist = scanconfig['_modulesenabled'].split(',')
if "sfp__stor_stdout" in modlist:
modlist.remove("sfp__stor_stdout")
targetType = sf.targetType(scantarget)
if targetType == None:
# Should never be triggered for a re-run scan..
return self.error("Invalid target type. Could not recognize it as " + \
"a human name, IP address, IP subnet, ASN, domain name or host name.")
# Start running a new scan
newId = sf.genScanInstanceGUID(scanname)
t = SpiderFootScanner(scanname, scantarget.lower(), targetType,
newId, modlist, cfg, modopts)
t.start()
# Wait until the scan has initialized
while globalScanStatus.getStatus(newId) == None:
print("[info] Waiting for the scan to initialize...")
time.sleep(1)
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(rerunscans=True,
docroot=self.docroot,
pageid="SCANLIST")
def test_attribute_status_should_return_status_as_a_string(self):
opts = self.default_options
opts['__modules__'] = dict()
scan_id = str(uuid.uuid4())
module_list = ['sfp__stor_db']
sfscan = SpiderFootScanner("example scan name", scan_id, "spiderfoot.net", "IP_ADDRESS", module_list, opts, start=False)
self.assertIsInstance(sfscan, SpiderFootScanner)
status = sfscan.status
self.assertIsInstance(status, str)
def test_init_argument_scanId_of_invalid_type_should_raise_TypeError(self):
"""
Test __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts, start=True)
"""
module_list = ['sfp__stor_db']
invalid_types = [None, list(), dict(), int()]
for invalid_type in invalid_types:
with self.subTest(invalid_type=invalid_type):
with self.assertRaises(TypeError):
SpiderFootScanner("example scan name", invalid_type, "spiderfoot.net", "IP_ADDRESS", module_list, self.default_options, start=False)
def test_init_argument_moduleList_of_invalid_type_should_raise_TypeError(self):
"""
Test __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts)
"""
scan_id = str(uuid.uuid4())
invalid_types = [None, "", dict(), int()]
for invalid_type in invalid_types:
with self.subTest(invalid_type=invalid_type):
with self.assertRaises(TypeError):
SpiderFootScanner("example scan name", scan_id, "spiderfoot.net", "IP_ADDRESS", invalid_type, self.default_options, start=False)
def rerunscan(self, id):
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
info = dbh.scanInstanceGet(id)
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if len(scanconfig) == 0:
return self.error("Something went wrong internally.")
modlist = scanconfig['_modulesenabled'].split(',')
targetType = sf.targetType(scantarget)
if targetType == None:
# It must then be a name, as a re-run scan should always have a clean
# target.
targetType = "HUMAN_NAME"
if targetType != "HUMAN_NAME":
scantarget = scantarget.lower()
# Start running a new scan
newId = sf.genScanInstanceGUID(scanname)
t = SpiderFootScanner(scanname, scantarget, targetType, newId,
modlist, cfg, modopts)
t.start()
# Wait until the scan has initialized
while globalScanStatus.getStatus(newId) == None:
print "[info] Waiting for the scan to initialize..."
time.sleep(1)
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
return templ.render(id=newId, name=unicode(scanname, 'utf-8', errors='replace'), docroot=self.docroot,
status=globalScanStatus.getStatus(newId), pageid="SCANLIST")
def test_init_argument_start_false_should_create_a_scan_without_starting_the_scan(self):
"""
Test __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts, start=True)
"""
opts = self.default_options
opts['__modules__'] = dict()
scan_id = str(uuid.uuid4())
module_list = ['sfp__stor_db']
sfscan = SpiderFootScanner("example scan name", scan_id, "spiderfoot.net", "IP_ADDRESS", module_list, opts, start=False)
self.assertIsInstance(sfscan, SpiderFootScanner)
self.assertEqual(sfscan.status, "INITIALIZING")
def test_init_argument_globalOpts_proxy_type_without_host_should_raise_ValueError(self):
"""
Test __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts)
"""
opts = self.default_options
opts['_socks1type'] = 'HTTP'
opts['__modules__'] = dict()
scan_id = str(uuid.uuid4())
module_list = ['sfp__stor_db']
with self.assertRaises(ValueError):
SpiderFootScanner("example scan name", scan_id, "spiderfoot.net", "IP_ADDRESS", module_list, opts, start=False)
def rerunscanmulti(self, ids):
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
for id in ids.split(","):
info = dbh.scanInstanceGet(id)
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if len(scanconfig) == 0:
return self.error("Something went wrong internally.")
modlist = scanconfig['_modulesenabled'].split(',')
targetType = sf.targetType(scantarget)
if targetType == None:
# Should never be triggered for a re-run scan..
return self.error("Invalid target type. Could not recognize it as " + \
"a human name, IP address, IP subnet, ASN, domain name or host name.")
# Start running a new scan
newId = sf.genScanInstanceGUID(scanname)
t = SpiderFootScanner(unicode(scanname, 'utf-8', errors='replace'),
unicode(scantarget, 'utf-8', errors='replace').lower(),
targetType, newId, modlist, cfg, modopts)
t.start()
# Wait until the scan has initialized
while globalScanStatus.getStatus(newId) == None:
print "[info] Waiting for the scan to initialize..."
time.sleep(1)
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(rerunscans=True, docroot=self.docroot, pageid="SCANLIST")
def startscan(self, scanname, scantarget, modulelist):
modopts = dict() # Not used yet as module options are set globally
[scanname, scantarget] = self.cleanUserInput([scanname, scantarget])
if scanname == "" or scantarget == "" or modulelist == "":
return self.error("Form incomplete.")
modlist = modulelist.replace('module_', '').split(',')
# For now we don't permit multiple simultaneous scans
for thread in threading.enumerate():
if thread.name.startswith("SF_"):
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(modules=self.config['__modules__'], alreadyRunning=True, runningScan=thread.name[3:])
# Start running a new scan
self.scanner = SpiderFootScanner(scanname, scantarget.lower(), modlist, self.config, modopts)
t = threading.Thread(name="SF_" + scanname, target=self.scanner.startScan)
t.start()
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(pageid='SCANLIST',newscan=scanname)
def test_init_argument_globalOpts_proxy_without_port_should_set_proxy(self):
"""
Test __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts)
"""
opts = self.default_options
opts['_socks1type'] = 'HTTP'
opts['_socks2addr'] = '127.0.0.1'
opts['_socks3port'] = ''
opts['__modules__'] = dict()
scan_id = str(uuid.uuid4())
module_list = ['sfp__stor_db']
SpiderFootScanner("example scan name", scan_id, "spiderfoot.net", "IP_ADDRESS", module_list, opts, start=False)
self.assertEqual('TBD', 'TBD')
def test_init_argument_globalOpts_of_invalid_type_should_raise_TypeError(
self):
"""
Test __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts)
"""
scan_id = str(uuid.uuid4())
module_list = ['sfp__stor_db']
invalid_types = [None, "", list(), int()]
for invalid_type in invalid_types:
with self.subTest(invalid_type=invalid_type):
with self.assertRaises(TypeError) as cm:
sfscan = SpiderFootScanner("example scan name",
scan_id,
"spiderfoot.net",
"IP_ADDRESS",
module_list,
invalid_type,
start=False)
def startscan(self, scanname, scantarget, modulelist):
modopts = dict() # Not used yet as module options are set globally
if scanname == "" or scantarget == "" or modulelist == "":
return self.error("Form incomplete.")
modlist = modulelist.replace('module_', '').split(',')
# For now we don't permit multiple simultaneous scans
for thread in threading.enumerate():
if thread.name.startswith("SF_"):
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(modules=self.config['__modules__'], alreadyRunning=True, runningScan=thread.name[3:])
# Start running a new scan
self.scanner = SpiderFootScanner(scanname, scantarget.lower(), modlist, self.config, modopts)
t = threading.Thread(name="SF_" + scanname, target=self.scanner.startScan)
t.start()
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(pageid='SCANLIST',newscan=scanname)
print(("[*] Modules enabled (" + str(len(modlist)) + "): " +
",".join(modlist)))
cfg = sf.configUnserialize(dbh.configGet(), sfConfig)
scanId = sf.genScanInstanceGUID(target)
# Debug mode is a variable that gets stored to the DB, so re-apply it
if args.debug:
cfg['_debug'] = True
else:
cfg['_debug'] = False
# If strict mode is enabled, filter the output from modules.
if args.x and args.t:
cfg['__outputfilter'] = args.t.split(",")
t = SpiderFootScanner(target, target, targetType, scanId, modlist, cfg,
dict())
t.daemon = True
t.start()
# If field headers weren't disabled, print them
if not args.H and args.o != "json":
if args.D:
delim = args.D
else:
if args.o in ["tab", None]:
delim = "\t"
if args.o == "csv":
delim = ","
if not args.r:
def startscan(self, scanname, scantarget, modulelist, typelist):
global globalScanStatus
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
types = dbh.eventTypes()
targetType = None
[scanname, scantarget] = self.cleanUserInput([scanname, scantarget])
if scanname == "" or scantarget == "":
return self.error("Form incomplete.")
if typelist == "" and modulelist == "":
return self.error("Form incomplete.")
if modulelist != "":
modlist = modulelist.replace('module_', '').split(',')
else:
typesx = typelist.replace('type_', '').split(',')
# 1. Find all modules that produce the requested types
modlist = sf.modulesProducing(typesx)
newmods = deepcopy(modlist)
newmodcpy = deepcopy(newmods)
# 2. For each type those modules consume, get modules producing
while len(newmodcpy) > 0:
for etype in sf.eventsToModules(newmodcpy):
xmods = sf.modulesProducing([etype])
for mod in xmods:
if mod not in modlist:
modlist.append(mod)
newmods.append(mod)
newmodcpy = deepcopy(newmods)
newmods = list()
# Add our mandatory storage module..
if "sfp__stor_db" not in modlist:
modlist.append("sfp__stor_db")
modlist.sort()
targetType = sf.targetType(scantarget)
if targetType is None:
return self.error("Invalid target type. Could not recognize it as " + \
"an IP address, IP subnet, domain name or host name.")
# Start running a new scan
scanId = sf.genScanInstanceGUID(scanname)
t = SpiderFootScanner(scanname, scantarget.lower(), targetType, scanId,
modlist, cfg, modopts)
t.start()
# Wait until the scan has initialized
while globalScanStatus.getStatus(scanId) is None:
print "[info] Waiting for the scan to initialize..."
time.sleep(1)
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
return templ.render(id=scanId, name=scanname, docroot=self.docroot,
status=globalScanStatus.getStatus(scanId), pageid="SCANLIST")
class SpiderFootWebUi:
lookup = TemplateLookup(directories=[''])
defaultConfig = dict()
config = dict()
scanner = None
def __init__(self, config):
self.defaultConfig = deepcopy(config)
dbh = SpiderFootDb(config)
# 'config' supplied will be the defaults, let's supplement them
# now with any configuration which may have previously been
# saved.
sf = SpiderFoot(config)
self.config = sf.configUnserialize(dbh.configGet(), config)
# Sanitize user input
def cleanUserInput(self, inputList):
ret = list()
for item in inputList:
ret.append(cgi.escape(item))
return ret
#
# USER INTERFACE PAGES
#
# Get result data in CSV format
def scaneventresultexport(self, id, type):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultEvent(id, type)
blob = "\"Updated\",\"Type\",\"Module\",\"Source\",\"Data\"\n"
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
escapedData = cgi.escape(row[1].replace("\n", "#LB#").replace("\r", "#LB#"))
escapedSrc = cgi.escape(row[2].replace("\n", "#LB#").replace("\r", "#LB#"))
blob = blob + "\"" + lastseen + "\",\"" + row[4] + "\",\""
blob = blob + row[3] + "\",\"" + escapedSrc + "\",\"" + escapedData + "\"\n"
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.csv"
cherrypy.response.headers['Content-Type'] = "application/csv"
cherrypy.response.headers['Pragma'] = "no-cache"
return blob
scaneventresultexport.exposed = True
# Configuration used for a scan
def scanopts(self, id):
ret = dict()
dbh = SpiderFootDb(self.config)
ret['config'] = dbh.scanConfigGet(id)
ret['configdesc'] = dict()
for key in ret['config'].keys():
if ':' not in key:
ret['configdesc'][key] = self.config['__globaloptdescs__'][key]
else:
[ modName, modOpt ] = key.split(':')
if not modName in self.config['__modules__'].keys():
continue
if not modOpt in self.config['__modules__'][modName]['optdescs'].keys():
continue
ret['configdesc'][key] = self.config['__modules__'][modName]['optdescs'][modOpt]
sf = SpiderFoot(self.config)
meta = dbh.scanInstanceGet(id)
if meta[3] != 0:
started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(meta[3]))
else:
started = "Not yet"
if meta[4] != 0:
finished = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(meta[4]))
else:
finished = "Not yet"
ret['meta'] = [meta[0], meta[1], meta[2], started, finished, meta[5]]
return json.dumps(ret)
scanopts.exposed = True
# Configure a new scan
def newscan(self):
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(pageid='NEWSCAN', modules=self.config['__modules__'])
newscan.exposed = True
# Main page listing scans available
def index(self):
# Look for referenced templates in the current directory only
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(pageid='SCANLIST')
index.exposed = True
# Information about a selected scan
def scaninfo(self, id):
dbh = SpiderFootDb(self.config)
res = dbh.scanInstanceGet(id)
if res == None:
return self.error("Scan ID not found.")
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
return templ.render(id=id, name=res[0], status=res[5], pageid='SCANLIST')
scaninfo.exposed = True
# Settings
def opts(self):
templ = Template(filename='dyn/opts.tmpl', lookup=self.lookup)
return templ.render(opts=self.config, pageid='SETTINGS')
opts.exposed = True
# Generic error, but not exposed as not called directly
def error(self, message):
templ = Template(filename='dyn/error.tmpl', lookup=self.lookup)
return templ.render(message=message)
# Delete a scan
def scandelete(self, id, confirm=None):
dbh = SpiderFootDb(self.config)
res = dbh.scanInstanceGet(id)
if res == None:
return self.error("Scan ID not found.")
if confirm != None:
dbh.scanInstanceDelete(id)
raise cherrypy.HTTPRedirect("/")
else:
templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup)
return templ.render(id=id, name=res[0])
scandelete.exposed = True
# Save settings, also used to completely reset them to default
def savesettings(self, allopts):
try:
dbh = SpiderFootDb(self.config)
# Reset config to default
if allopts == "RESET":
dbh.configClear() # Clear it in the DB
self.config = deepcopy(self.defaultConfig) # Clear in memory
else:
useropts = json.loads(allopts)
currentopts = deepcopy(self.config)
# Make a new config where the user options override
# the current system config.
sf = SpiderFoot(self.config)
self.config = sf.configUnserialize(useropts, currentopts)
dbh.configSet(sf.configSerialize(currentopts))
except Exception as e:
return self.error("Processing one or more of your inputs failed: " + str(e))
templ = Template(filename='dyn/opts.tmpl', lookup=self.lookup)
return templ.render(opts=self.config, pageid='SETTINGS', updated=True)
savesettings.exposed = True
# Initiate a scan
def startscan(self, scanname, scantarget, modulelist):
modopts = dict() # Not used yet as module options are set globally
[scanname, scantarget] = self.cleanUserInput([scanname, scantarget])
if scanname == "" or scantarget == "" or modulelist == "":
return self.error("Form incomplete.")
modlist = modulelist.replace('module_', '').split(',')
# For now we don't permit multiple simultaneous scans
for thread in threading.enumerate():
if thread.name.startswith("SF_"):
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(modules=self.config['__modules__'], alreadyRunning=True, runningScan=thread.name[3:])
# Start running a new scan
self.scanner = SpiderFootScanner(scanname, scantarget.lower(), modlist, self.config, modopts)
t = threading.Thread(name="SF_" + scanname, target=self.scanner.startScan)
t.start()
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(pageid='SCANLIST',newscan=scanname)
startscan.exposed = True
# Stop a scan (id variable is unnecessary for now given that only one simultaneous
# scan is permitted.)
def stopscan(self, id):
if self.scanner == None:
return self.error("There are no scans running. A data consistency error for this scan probably exists. <a href='/scandelete?id=" + id + "&confirm=1'>Click here to delete it.</a>")
if self.scanner.scanStatus(id) == "ABORTED":
return self.error("The scan is already aborted.")
if not self.scanner.scanStatus(id) == "RUNNING":
return self.error("The running scan is currently in the state '" + self.scanner.scanStatus(id) + "', please try again later or restart SpiderFoot.")
self.scanner.stopScan(id)
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(pageid='SCANLIST',stoppedscan=True)
stopscan.exposed = True
#
# DATA PROVIDERS
#
# Scan log data
def scanlog(self, id):
dbh = SpiderFootDb(self.config)
data = dbh.scanLogs(id)
retdata = []
for row in data:
generated = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]/1000))
retdata.append([generated, row[1], row[2], cgi.escape(row[3])])
return json.dumps(retdata)
scanlog.exposed = True
# Produce a list of scans
def scanlist(self):
dbh = SpiderFootDb(self.config)
data = dbh.scanInstanceList()
retdata = []
for row in data:
created = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[3]))
if row[4] != 0:
started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[4]))
else:
started = "Not yet"
if row[5] != 0:
finished = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[5]))
else:
finished = "Not yet"
retdata.append([row[0], row[1], row[2], created, started, finished, row[6], row[7]])
return json.dumps(retdata)
scanlist.exposed = True
# Summary of scan results
def scansummary(self, id):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultSummary(id)
retdata = []
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[2]))
retdata.append([row[0], row[1], lastseen, row[3], row[4]])
return json.dumps(retdata)
scansummary.exposed = True
# Event results for a scan
def scaneventresults(self, id, eventType):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultEvent(id, eventType)
retdata = []
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
escaped = cgi.escape(row[1])
# For debug
retdata.append([lastseen, escaped, row[2], row[3], row[5], row[6], row[7], row[8]])
#retdata.append([lastseen, escaped, row[2], row[3], row[5], row[6], row[7]])
return json.dumps(retdata, ensure_ascii=False)
scaneventresults.exposed = True
# Unique event results for a scan
def scaneventresultsunique(self, id, eventType):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultEventUnique(id, eventType)
retdata = []
for row in data:
escaped = cgi.escape(row[0])
retdata.append([escaped, row[1]])
return json.dumps(retdata, ensure_ascii=False)
scaneventresultsunique.exposed = True
# Historical data for the scan, graphs will be rendered in JS
def scanhistory(self, id):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultHistory(id)
return json.dumps(data, ensure_ascii=False)
scanhistory.exposed = True
class SpiderFootWebUi:
lookup = TemplateLookup(directories=[''])
defaultConfig = dict()
config = dict()
scanner = None
token = None
def __init__(self, config):
self.defaultConfig = deepcopy(config)
dbh = SpiderFootDb(config)
# 'config' supplied will be the defaults, let's supplement them
# now with any configuration which may have previously been
# saved.
sf = SpiderFoot(config)
self.config = sf.configUnserialize(dbh.configGet(), config)
if self.config['__webaddr'] == "0.0.0.0":
addr = "<IP of this host>"
else:
addr = self.config['__webaddr']
print ""
print ""
print "*************************************************************"
print " Use SpiderFoot by starting your web browser of choice and "
print " browse to http://" + addr + ":" + str(self.config['__webport'])
print "*************************************************************"
print ""
print ""
# Sanitize user input
def cleanUserInput(self, inputList):
ret = list()
for item in inputList:
c = cgi.escape(item, True)
c = c.replace('\'', '"')
ret.append(c)
return ret
#
# USER INTERFACE PAGES
#
# Get result data in CSV format
def scaneventresultexport(self, id, type, dialect="excel"):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultEvent(id, type)
fileobj = StringIO()
parser = csv.writer(fileobj, dialect=dialect)
parser.writerow(["Updated", "Type", "Module", "Source", "Data"])
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
datafield = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "")
parser.writerow([lastseen, str(row[4]), str(row[3]), str(row[2]), datafield])
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.csv"
cherrypy.response.headers['Content-Type'] = "application/csv"
cherrypy.response.headers['Pragma'] = "no-cache"
return fileobj.getvalue()
scaneventresultexport.exposed = True
# Configuration used for a scan
def scanopts(self, id):
ret = dict()
dbh = SpiderFootDb(self.config)
ret['config'] = dbh.scanConfigGet(id)
ret['configdesc'] = dict()
for key in ret['config'].keys():
if ':' not in key:
ret['configdesc'][key] = self.config['__globaloptdescs__'][key]
else:
[ modName, modOpt ] = key.split(':')
if not modName in self.config['__modules__'].keys():
continue
if not modOpt in self.config['__modules__'][modName]['optdescs'].keys():
continue
ret['configdesc'][key] = self.config['__modules__'][modName]['optdescs'][modOpt]
sf = SpiderFoot(self.config)
meta = dbh.scanInstanceGet(id)
if meta[3] != 0:
started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(meta[3]))
else:
started = "Not yet"
if meta[4] != 0:
finished = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(meta[4]))
else:
finished = "Not yet"
ret['meta'] = [meta[0], meta[1], meta[2], started, finished, meta[5]]
return json.dumps(ret)
scanopts.exposed = True
# Configure a new scan
def newscan(self):
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(pageid='NEWSCAN', types=types,
modules=self.config['__modules__'])
newscan.exposed = True
# Main page listing scans available
def index(self):
# Look for referenced templates in the current directory only
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(pageid='SCANLIST')
index.exposed = True
# Information about a selected scan
def scaninfo(self, id):
dbh = SpiderFootDb(self.config)
res = dbh.scanInstanceGet(id)
if res == None:
return self.error("Scan ID not found.")
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
return templ.render(id=id, name=res[0], status=res[5],
pageid="SCANLIST")
scaninfo.exposed = True
# Settings
def opts(self):
templ = Template(filename='dyn/opts.tmpl', lookup=self.lookup)
self.token = random.randint(0, 99999999)
return templ.render(opts=self.config, pageid='SETTINGS', token=self.token)
opts.exposed = True
# Generic error, but not exposed as not called directly
def error(self, message):
templ = Template(filename='dyn/error.tmpl', lookup=self.lookup)
return templ.render(message=message)
# Delete a scan
def scandelete(self, id, confirm=None):
dbh = SpiderFootDb(self.config)
res = dbh.scanInstanceGet(id)
if res == None:
return self.error("Scan ID not found.")
if confirm != None:
dbh.scanInstanceDelete(id)
raise cherrypy.HTTPRedirect("/")
else:
templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup)
return templ.render(id=id, name=res[0], pageid="SCANLIST")
scandelete.exposed = True
# Save settings, also used to completely reset them to default
def savesettings(self, allopts, token):
if str(token) != str(self.token):
return self.error("Invalid token (" + str(self.token) + ").")
try:
dbh = SpiderFootDb(self.config)
# Reset config to default
if allopts == "RESET":
dbh.configClear() # Clear it in the DB
self.config = deepcopy(self.defaultConfig) # Clear in memory
else:
useropts = json.loads(allopts)
cleanopts = dict()
for opt in useropts.keys():
cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0]
currentopts = deepcopy(self.config)
# Make a new config where the user options override
# the current system config.
sf = SpiderFoot(self.config)
self.config = sf.configUnserialize(cleanopts, currentopts)
dbh.configSet(sf.configSerialize(currentopts))
except Exception as e:
return self.error("Processing one or more of your inputs failed: " + str(e))
templ = Template(filename='dyn/opts.tmpl', lookup=self.lookup)
self.token = random.randint(0, 99999999)
return templ.render(opts=self.config, pageid='SETTINGS', updated=True,
token=self.token)
savesettings.exposed = True
# Initiate a scan
def startscan(self, scanname, scantarget, modulelist, typelist):
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(self.config)
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
[scanname, scantarget] = self.cleanUserInput([scanname, scantarget])
if scanname == "" or scantarget == "":
return self.error("Form incomplete.")
if typelist == "" and modulelist == "":
return self.error("Form incomplete.")
if modulelist != "":
modlist = modulelist.replace('module_', '').split(',')
else:
typesx = typelist.replace('type_', '').split(',')
# 1. Find all modules that produce the requested types
modlist = sf.modulesProducing(typesx)
newmods = deepcopy(modlist)
newmodcpy = deepcopy(newmods)
# 2. For each type those modules consume, get modules producing
while len(newmodcpy) > 0:
for etype in sf.eventsToModules(newmodcpy):
xmods = sf.modulesProducing([etype])
for mod in xmods:
if mod not in modlist:
modlist.append(mod)
newmods.append(mod)
newmodcpy = deepcopy(newmods)
newmods = list()
# Add our mandatory storage module..
if "sfp__stor_db" not in modlist:
modlist.append("sfp__stor_db")
modlist.sort()
# For now we don't permit multiple simultaneous scans
for thread in threading.enumerate():
if thread.name.startswith("SF_"):
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(modules=self.config['__modules__'],
alreadyRunning=True, runningScan=thread.name[3:],
types=types, pageid="NEWSCAN")
# Start running a new scan
self.scanner = SpiderFootScanner(scanname, scantarget.lower(), modlist,
self.config, modopts)
t = threading.Thread(name="SF_" + scanname, target=self.scanner.startScan)
t.start()
# Spin cycles waiting for the scan ID to be set
while self.scanner.myId == None:
time.sleep(1)
continue
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
return templ.render(id=self.scanner.myId, name=scanname,
status=self.scanner.status, pageid="SCANLIST")
startscan.exposed = True
# Stop a scan (id variable is unnecessary for now given that only one simultaneous
# scan is permitted.)
def stopscan(self, id):
if self.scanner == None:
return self.error("There are no scans running. A data consistency " + \
"error for this scan probably exists. <a href='/scandelete?id=" + \
id + "&confirm=1'>Click here to delete it.</a>")
if self.scanner.scanStatus(id) == "ABORTED":
return self.error("The scan is already aborted.")
if not self.scanner.scanStatus(id) == "RUNNING":
return self.error("The running scan is currently in the state '" + \
self.scanner.scanStatus(id) + "', please try again later or restart " + \
" SpiderFoot.")
self.scanner.stopScan(id)
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(pageid='SCANLIST',stoppedscan=True)
stopscan.exposed = True
#
# DATA PROVIDERS
#
# Scan log data
def scanlog(self, id, limit=None):
dbh = SpiderFootDb(self.config)
data = dbh.scanLogs(id, limit)
retdata = []
for row in data:
generated = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]/1000))
retdata.append([generated, row[1], row[2],
cgi.escape(unicode(row[3], errors='replace'))])
return json.dumps(retdata)
scanlog.exposed = True
# Scan error data
def scanerrors(self, id, limit=None):
dbh = SpiderFootDb(self.config)
data = dbh.scanErrors(id, limit)
retdata = []
for row in data:
generated = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]/1000))
retdata.append([generated, row[1],
cgi.escape(unicode(row[2], errors='replace'))])
return json.dumps(retdata)
scanerrors.exposed = True
# Produce a list of scans
def scanlist(self):
dbh = SpiderFootDb(self.config)
data = dbh.scanInstanceList()
retdata = []
for row in data:
created = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[3]))
if row[4] != 0:
started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[4]))
else:
started = "Not yet"
if row[5] != 0:
finished = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[5]))
else:
finished = "Not yet"
retdata.append([row[0], row[1], row[2], created, started, finished, row[6], row[7]])
return json.dumps(retdata)
scanlist.exposed = True
# Basic information about a scan
def scanstatus(self, id):
dbh = SpiderFootDb(self.config)
data = dbh.scanInstanceGet(id)
created = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[2]))
started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[3]))
ended = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[4]))
retdata = [data[0], data[1], created, started, ended, data[5]]
return json.dumps(retdata)
scanstatus.exposed = True
# Summary of scan results
def scansummary(self, id):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultSummary(id)
retdata = []
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[2]))
retdata.append([row[0], row[1], lastseen, row[3], row[4]])
return json.dumps(retdata)
scansummary.exposed = True
# Event results for a scan
def scaneventresults(self, id, eventType):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultEvent(id, eventType)
retdata = []
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
escapeddata = cgi.escape(row[1])
escapedsrc = cgi.escape(row[2])
retdata.append([lastseen, escapeddata, escapedsrc,
row[3], row[5], row[6], row[7], row[8]])
return json.dumps(retdata, ensure_ascii=False)
scaneventresults.exposed = True
# Unique event results for a scan
def scaneventresultsunique(self, id, eventType):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultEventUnique(id, eventType)
retdata = []
for row in data:
escaped = cgi.escape(row[0])
retdata.append([escaped, row[1], row[2]])
return json.dumps(retdata, ensure_ascii=False)
scaneventresultsunique.exposed = True
# Historical data for the scan, graphs will be rendered in JS
def scanhistory(self, id):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultHistory(id)
return json.dumps(data, ensure_ascii=False)
scanhistory.exposed = True
def scanelementtypediscovery(self, id, eventType):
keepGoing = True
sf = SpiderFoot(self.config)
dbh = SpiderFootDb(self.config)
pc = dict()
datamap = dict()
# Get the events we will be tracing back from
leafSet = dbh.scanResultEvent(id, eventType)
# Get the first round of source IDs for the leafs
nextIds = list()
for row in leafSet:
# these must be unique values!
parentId = row[9]
childId = row[8]
datamap[childId] = row
if pc.has_key(parentId):
if childId not in pc[parentId]:
pc[parentId].append(childId)
else:
pc[parentId] = [ childId ]
# parents of the leaf set
if parentId not in nextIds:
nextIds.append(parentId)
while keepGoing:
#print "Next IDs: " + str(nextIds)
parentSet = dbh.scanElementSources(id, nextIds)
nextIds = list()
keepGoing = False
for row in parentSet:
parentId = row[9]
childId = row[8]
datamap[childId] = row
# Prevent us from looping at root
# 0 = event_hash and 3 = source_event_hash
if row[8] == "ROOT" and row[9] == "ROOT":
continue
if pc.has_key(parentId):
if childId not in pc[parentId]:
pc[parentId].append(childId)
else:
pc[parentId] = [ childId ]
if parentId not in nextIds:
nextIds.append(parentId)
# Stop until we've found ROOT
# 3 = source_event_hash
if row[3] != "ROOT":
keepGoing = True
#print pc
retdata = dict()
retdata['tree'] = sf.dataParentChildToTree(pc)
retdata['data'] = datamap
return json.dumps(retdata, ensure_ascii=False)
scanelementtypediscovery.exposed = True
def startscan(self, scanname, scantarget, modulelist, typelist):
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(self.config)
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
[scanname, scantarget] = self.cleanUserInput([scanname, scantarget])
if scanname == "" or scantarget == "":
return self.error("Form incomplete.")
if typelist == "" and modulelist == "":
return self.error("Form incomplete.")
if modulelist != "":
modlist = modulelist.replace('module_', '').split(',')
else:
typesx = typelist.replace('type_', '').split(',')
# 1. Find all modules that produce the requested types
modlist = sf.modulesProducing(typesx)
newmods = deepcopy(modlist)
newmodcpy = deepcopy(newmods)
# 2. For each type those modules consume, get modules producing
while len(newmodcpy) > 0:
for etype in sf.eventsToModules(newmodcpy):
xmods = sf.modulesProducing([etype])
for mod in xmods:
if mod not in modlist:
modlist.append(mod)
newmods.append(mod)
newmodcpy = deepcopy(newmods)
newmods = list()
# Add our mandatory storage module..
if "sfp__stor_db" not in modlist:
modlist.append("sfp__stor_db")
modlist.sort()
# For now we don't permit multiple simultaneous scans
for thread in threading.enumerate():
if thread.name.startswith("SF_"):
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(modules=self.config['__modules__'],
alreadyRunning=True, runningScan=thread.name[3:],
types=types, pageid="NEWSCAN")
# Start running a new scan
self.scanner = SpiderFootScanner(scanname, scantarget.lower(), modlist,
self.config, modopts)
t = threading.Thread(name="SF_" + scanname, target=self.scanner.startScan)
t.start()
# Spin cycles waiting for the scan ID to be set
while self.scanner.myId == None:
time.sleep(1)
continue
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
return templ.render(id=self.scanner.myId, name=scanname,
status=self.scanner.status, pageid="SCANLIST")
def startscan(self, scanname, scantarget, modulelist, typelist):
global globalScanStatus
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
types = dbh.eventTypes()
targetType = None
[scanname, scantarget] = self.cleanUserInput([scanname, scantarget])
if scanname == "" or scantarget == "":
return self.error("Form incomplete.")
if typelist == "" and modulelist == "":
return self.error("Form incomplete.")
if modulelist != "":
modlist = modulelist.replace('module_', '').split(',')
else:
typesx = typelist.replace('type_', '').split(',')
# 1. Find all modules that produce the requested types
modlist = sf.modulesProducing(typesx)
newmods = deepcopy(modlist)
newmodcpy = deepcopy(newmods)
# 2. For each type those modules consume, get modules producing
while len(newmodcpy) > 0:
for etype in sf.eventsToModules(newmodcpy):
xmods = sf.modulesProducing([etype])
for mod in xmods:
if mod not in modlist:
modlist.append(mod)
newmods.append(mod)
newmodcpy = deepcopy(newmods)
newmods = list()
# Add our mandatory storage module..
if "sfp__stor_db" not in modlist:
modlist.append("sfp__stor_db")
modlist.sort()
regexToType = {
"^\d+\.\d+\.\d+\.\d+$": "IP_ADDRESS",
"^\d+\.\d+\.\d+\.\d+/\d+$": "NETBLOCK_OWNER",
"^.[a-zA-Z\-0-9\.]+$": "INTERNET_NAME"
}
# Parse the target and set the targetType
for rx in regexToType.keys():
if re.match(rx, scantarget, re.IGNORECASE):
targetType = regexToType[rx]
break
if targetType == None:
return self.error("Invalid target type. Could not recognize it as " + \
"an IP address, IP subnet, domain name or host name.")
# Start running a new scan
scanId = sf.genScanInstanceGUID(scanname)
t = SpiderFootScanner(scanname, scantarget.lower(), targetType, scanId,
modlist, cfg, modopts)
t.start()
# Wait until the scan has initialized
while globalScanStatus.getStatus(scanId) == None:
print "[info] Waiting for the scan to initialize..."
time.sleep(1)
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
return templ.render(id=scanId, name=scanname,
status=globalScanStatus.getStatus(scanId), pageid="SCANLIST")
def startscan(self, scanname, scantarget, modulelist, typelist, usecase):
global globalScanStatus
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
types = dbh.eventTypes()
targetType = None
[scanname, scantarget] = self.cleanUserInput([scanname, scantarget])
if scanname == "" or scantarget == "":
return self.error("Form incomplete.")
if typelist == "" and modulelist == "" and usecase == "":
return self.error("Form incomplete.")
# User selected modules
if modulelist != "":
modlist = modulelist.replace('module_', '').split(',')
# User selected types
if len(modlist) == 0 and typelist != "":
typesx = typelist.replace('type_', '').split(',')
# 1. Find all modules that produce the requested types
modlist = sf.modulesProducing(typesx)
newmods = deepcopy(modlist)
newmodcpy = deepcopy(newmods)
# 2. For each type those modules consume, get modules producing
while len(newmodcpy) > 0:
for etype in sf.eventsToModules(newmodcpy):
xmods = sf.modulesProducing([etype])
for mod in xmods:
if mod not in modlist:
modlist.append(mod)
newmods.append(mod)
newmodcpy = deepcopy(newmods)
newmods = list()
# User selected a use case
if len(modlist) == 0 and usecase != "":
for mod in self.config['__modules__']:
if usecase == 'all' or usecase in self.config['__modules__'][
mod]['cats']:
modlist.append(mod)
# Add our mandatory storage module..
if "sfp__stor_db" not in modlist:
modlist.append("sfp__stor_db")
modlist.sort()
targetType = sf.targetType(scantarget)
if targetType is None:
return self.error("Invalid target type. Could not recognize it as " + \
"an IP address, IP subnet, domain name or host name.")
# Start running a new scan
scanId = sf.genScanInstanceGUID(scanname)
t = SpiderFootScanner(scanname, scantarget.lower(), targetType, scanId,
modlist, cfg, modopts)
t.start()
# Wait until the scan has initialized
while globalScanStatus.getStatus(scanId) is None:
print "[info] Waiting for the scan to initialize..."
time.sleep(1)
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
return templ.render(id=scanId,
name=scanname,
docroot=self.docroot,
status=globalScanStatus.getStatus(scanId),
pageid="SCANLIST")
if not args.q:
print "[*] Modules enabled (" + str(len(modlist)) + "): " + ",".join(modlist)
cfg = sf.configUnserialize(dbh.configGet(), sfConfig)
scanId = sf.genScanInstanceGUID(target)
# Debug mode is a variable that gets stored to the DB, so re-apply it
if args.debug:
cfg['_debug'] = True
else:
cfg['_debug'] = False
# If strict mode is enabled, filter the output from modules.
if args.x and args.t:
cfg['__outputfilter'] = args.t.split(",")
t = SpiderFootScanner(target, target, targetType, scanId,
modlist, cfg, dict())
t.daemon = True
t.start()
while True:
info = dbh.scanInstanceGet(scanId)
if not info:
time.sleep(1)
continue
if info[5] in [ "ERROR-FAILED", "ABORT-REQUESTED", "ABORTED", "FINISHED" ]:
if not args.q:
print "[*] Scan completed with status " + info[5]
sys.exit(0)
time.sleep(1)
sys.exit(0)