def rerunscan(self, id):
"""Rerun a scan
Args:
id (str): scan ID
"""
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
info = dbh.scanInstanceGet(id)
if not info:
return self.error("Invalid scan ID.")
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if len(scanconfig) == 0:
return self.error("Something went wrong internally.")
modlist = scanconfig['_modulesenabled'].split(',')
if "sfp__stor_stdout" in modlist:
modlist.remove("sfp__stor_stdout")
targetType = sf.targetType(scantarget)
if targetType is None:
# It must then be a name, as a re-run scan should always have a clean
# target. Put quotes around the target value and try to determine the
# target type again.
targetType = sf.targetType(f'"{scantarget}"')
if targetType != "HUMAN_NAME":
scantarget = scantarget.lower()
# Start running a new scan
scanId = sf.genScanInstanceGUID()
try:
p = mp.Process(target=SpiderFootScanner,
args=(scanname, scanId, scantarget, targetType,
modlist, cfg))
p.daemon = True
p.start()
except BaseException as e:
print("[-] Scan [%s] failed: %s" % (scanId, e))
return self.error("Scan [%s] failed: %s" % (scanId, e))
# Wait until the scan has initialized
while dbh.scanInstanceGet(scanId) is None:
print("[info] Waiting for the scan to initialize...")
time.sleep(1)
raise cherrypy.HTTPRedirect(f"scaninfo?id={scanId}", status=302)
def rerunscanmulti(self, ids):
"""Rerun scans
Args:
ids (str): comma separated list of scan IDs
"""
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
for id in ids.split(","):
info = dbh.scanInstanceGet(id)
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if len(scanconfig) == 0:
return self.error("Something went wrong internally.")
modlist = scanconfig['_modulesenabled'].split(',')
if "sfp__stor_stdout" in modlist:
modlist.remove("sfp__stor_stdout")
targetType = sf.targetType(scantarget)
if targetType is None:
# Should never be triggered for a re-run scan..
return self.error(
"Invalid target type. Could not recognize it as a human name, IP address, IP subnet, ASN, domain name or host name."
)
# Start running a new scan
scanId = sf.genScanInstanceGUID()
try:
p = mp.Process(target=SpiderFootScanner,
args=(scanname, scanId, scantarget, targetType,
modlist, cfg))
p.daemon = True
p.start()
except BaseException as e:
print("[-] Scan [%s] failed: %s" % (scanId, e))
return self.error("Scan [%s] failed: %s" % (scanId, e))
# Wait until the scan has initialized
while dbh.scanInstanceGet(scanId) is None:
print("[info] Waiting for the scan to initialize...")
time.sleep(1)
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(rerunscans=True,
docroot=self.docroot,
pageid="SCANLIST")
def rerunscan(self, id):
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
info = dbh.scanInstanceGet(id)
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if len(scanconfig) == 0:
return self.error("Something went wrong internally.")
modlist = scanconfig['_modulesenabled'].split(',')
if "sfp__stor_stdout" in modlist:
modlist.remove("sfp__stor_stdout")
targetType = sf.targetType(scantarget)
if targetType is None:
# It must then be a name, as a re-run scan should always have a clean
# target. Put quotes around the target value and try to determine the
# target type again.
targetType = sf.targetType(f'"{scantarget}"')
if targetType != "HUMAN_NAME":
scantarget = scantarget.lower()
# Start running a new scan
newId = sf.genScanInstanceGUID(scanname)
p = mp.Process(target=SpiderFootScanner,
args=(scanname, scantarget, targetType, newId, modlist,
cfg, modopts))
p.start()
# Wait until the scan has initialized
while dbh.scanInstanceGet(newId) == None:
print("[info] Waiting for the scan to initialize...")
time.sleep(1)
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
return templ.render(id=newId,
name=str(scanname),
docroot=self.docroot,
status=dbh.scanInstanceGet(newId),
pageid="SCANLIST")
def scansummary(self, id, by):
"""Summary of scan results
Args:
id (str): scan ID
by: TBD
"""
retdata = []
dbh = SpiderFootDb(self.config)
try:
scandata = dbh.scanResultSummary(id, by)
except Exception:
return json.dumps(retdata)
try:
statusdata = dbh.scanInstanceGet(id)
except Exception:
return json.dumps(retdata)
for row in scandata:
if row[0] == "ROOT":
continue
lastseen = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[2]))
retdata.append(
[row[0], row[1], lastseen, row[3], row[4], statusdata[5]])
return json.dumps(retdata)
def stopscanmulti(self, ids):
"""Stop a scan
Args:
ids (str): comma separated list of scan IDs
Note:
Unnecessary for now given that only one simultaneous scan is permitted
"""
dbh = SpiderFootDb(self.config)
error = list()
for id in ids.split(","):
scaninfo = dbh.scanInstanceGet(id)
if not scaninfo:
return self.error("Invalid scan ID: %s" % id)
scanname = str(scaninfo[0])
scanstatus = scaninfo[5]
if scanstatus == "FINISHED":
error.append(
"Scan '%s' is in a finished state. <a href='/scandelete?id=%s&confirm=1'>Maybe you want to delete it instead?</a>"
% (scanname, id))
continue
if scanstatus == "ABORTED":
error.append("Scan '" + scanname + "' is already aborted.")
continue
dbh.scanInstanceSet(id, status="ABORT-REQUESTED")
raise cherrypy.HTTPRedirect("/")
def scaneventresultexportmulti(self, ids, dialect="excel"):
dbh = SpiderFootDb(self.config)
scaninfo = dict()
data = list()
for id in ids.split(','):
scaninfo[id] = dbh.scanInstanceGet(id)
data = data + dbh.scanResultEvent(id)
fileobj = StringIO()
parser = csv.writer(fileobj, dialect=dialect)
parser.writerow([
"Scan Name", "Updated", "Type", "Module", "Source", "F/P", "Data"
])
for row in data:
if row[4] == "ROOT":
continue
lastseen = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0]))
datafield = str(row[1]).replace("<SFURL>",
"").replace("</SFURL>", "")
parser.writerow([
scaninfo[row[12]][0], lastseen,
str(row[4]),
str(row[3]),
str(row[2]), row[13], datafield
])
cherrypy.response.headers[
'Content-Disposition'] = "attachment; filename=SpiderFoot.csv"
cherrypy.response.headers['Content-Type'] = "application/csv"
cherrypy.response.headers['Pragma'] = "no-cache"
return fileobj.getvalue()
def clonescan(self, id):
sf = SpiderFoot(self.config)
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
info = dbh.scanInstanceGet(id)
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if scanname == "" or scantarget == "" or len(scanconfig) == 0:
return self.error("Something went wrong internally.")
targetType = sf.targetType(scantarget)
if targetType == None:
# It must be a name, so wrap quotes around it
scantarget = """ + scantarget + """
modlist = scanconfig['_modulesenabled'].split(',')
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(pageid='NEWSCAN', types=types, docroot=self.docroot,
modules=self.config['__modules__'], selectedmods=modlist,
scanname=unicode(scanname, 'utf-8', errors='replace'),
scantarget=unicode(scantarget, 'utf-8', errors='replace'))
def scandeletemulti(self, ids, confirm=None):
dbh = SpiderFootDb(self.config)
names = list()
for id in ids.split(','):
res = dbh.scanInstanceGet(id)
names.append(res[0])
if res is None:
return self.error("Scan ID not found (" + id + ").")
if res[5] in ["RUNNING", "STARTING", "STARTED"]:
return self.error("You cannot delete running scans.")
if confirm is not None:
for id in ids.split(','):
dbh.scanInstanceDelete(id)
raise cherrypy.HTTPRedirect("/")
else:
templ = Template(filename='dyn/scandelete.tmpl',
lookup=self.lookup)
return templ.render(id=None,
name=None,
ids=ids.split(','),
names=names,
pageid="SCANLIST",
docroot=self.docroot)
def resultsetfp(self, id, resultids, fp):
dbh = SpiderFootDb(self.config)
if fp not in ["0", "1"]:
return json.dumps(
["ERROR", "No FP flag set or not set correctly."])
ids = json.loads(resultids)
if not ids:
return json.dumps(["ERROR", "No IDs supplied."])
# Cannot set FPs if a scan is not completed
status = dbh.scanInstanceGet(id)
if status[5] not in ["ABORTED", "FINISHED", "ERROR-FAILED"]:
return json.dumps(["WARNING", "Scan must be in a finished state when " + \
"setting False Positives."])
# Make sure the user doesn't set something as non-FP when the
# parent is set as an FP.
if fp == "0":
data = dbh.scanElementSourcesDirect(id, ids)
for row in data:
if str(row[14]) == "1":
return json.dumps(["WARNING",
"You cannot unset an element as False Positive " + \
"if a parent element is still False Positive."])
# Set all the children as FPs too.. it's only logical afterall, right?
childs = dbh.scanElementChildrenAll(id, ids)
allIds = ids + childs
ret = dbh.scanResultsUpdateFP(id, allIds, fp)
if not ret:
return json.dumps(["ERROR", "Exception encountered."])
else:
return json.dumps(["SUCCESS", ""])
def scanexportjsonmulti(self, ids):
dbh = SpiderFootDb(self.config)
scaninfo = dict()
for id in ids.split(','):
scan_name = dbh.scanInstanceGet(id)[0]
if scan_name not in scaninfo:
scaninfo[scan_name] = []
for row in dbh.scanResultEvent(id):
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
event_data = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "")
source_data = str(row[2])
source_module = str(row[3])
event_type = row[4]
false_positive = row[13]
if event_type == "ROOT":
continue
scaninfo[scan_name].append({
"data": event_data,
"type": event_type,
"source_module": source_module,
"source_data": source_data,
"false_positive": false_positive,
"lastseen": lastseen
})
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.json"
cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"
cherrypy.response.headers['Pragma'] = "no-cache"
return json.dumps(scaninfo)
def stopscanmulti(self, ids):
global globalScanStatus # running scans
dbh = SpiderFootDb(self.config)
error = list()
for id in ids.split(","):
errState = False
scaninfo = dbh.scanInstanceGet(id)
if globalScanStatus.getStatus(
id) == "FINISHED" or scaninfo[5] == "FINISHED":
error.append("Scan '" + scaninfo[0] + "' is in a finished state. <a href='/scandelete?id=" + \
id + "&confirm=1'>Maybe you want to delete it instead?</a>")
errState = True
if not errState and (globalScanStatus.getStatus(id) == "ABORTED"
or scaninfo[5] == "ABORTED"):
error.append("Scan '" + scaninfo[0] + "' is already aborted.")
errState = True
if not errState and globalScanStatus.getStatus(id) is None:
error.append("Scan '" + scaninfo[0] + "' is not actually running. A data consistency " + \
"error for this scan probably exists. <a href='/scandelete?id=" + \
id + "&confirm=1'>Click here to delete it.</a>")
errState = True
if not errState:
globalScanStatus.setStatus(id, "ABORT-REQUESTED")
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(pageid='SCANLIST',
stoppedscan=True,
errors=error,
docroot=self.docroot)
def test_scanInstanceGet_should_return_scan_info(self):
"""
Test scanInstanceGet(self, instanceId)
"""
sfdb = SpiderFootDb(self.default_options, False)
instance_id = "example instance id"
scan_name = "example scan name"
scan_target = "example scan target"
sfdb.scanInstanceCreate(instance_id, scan_name, scan_target)
scan_instance_get = sfdb.scanInstanceGet(instance_id)
self.assertEqual(len(scan_instance_get), 6)
self.assertIsInstance(scan_instance_get[0], str)
self.assertEqual(scan_instance_get[0], scan_name)
self.assertIsInstance(scan_instance_get[1], str)
self.assertEqual(scan_instance_get[1], scan_target)
self.assertIsInstance(scan_instance_get[2], float)
self.assertIsInstance(scan_instance_get[3], float)
self.assertIsInstance(scan_instance_get[4], float)
self.assertIsInstance(scan_instance_get[5], str)
self.assertEqual(scan_instance_get[5], 'CREATED')
def scandelete(self, id, confirm=None, raw=False):
"""Delete a scan
Args:
id (str): scan ID
confirm: TBD
raw: TBD
"""
dbh = SpiderFootDb(self.config)
res = dbh.scanInstanceGet(id)
if res is None:
if raw:
return json.dumps(["ERROR", "Scan ID not found."])
return self.error("Scan ID not found.")
if confirm:
dbh.scanInstanceDelete(id)
if raw:
return json.dumps(["SUCCESS", ""])
raise cherrypy.HTTPRedirect("/")
templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup)
return templ.render(id=id,
name=str(res[0]),
names=list(),
ids=list(),
pageid="SCANLIST",
docroot=self.docroot)
def scanexportjsonmulti(self, ids):
dbh = SpiderFootDb(self.config)
scaninfo = dict()
for id in ids.split(','):
scan_name = dbh.scanInstanceGet(id)[0]
if scan_name not in scaninfo:
scaninfo[scan_name] = []
for row in dbh.scanResultEvent(id):
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
event_data = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "")
source_data = str(row[2])
source_module = str(row[3])
event_type = row[4]
false_positive = row[13]
if event_type == "ROOT":
continue
scaninfo[scan_name].append({
"data": event_data,
"type": event_type,
"source_module": source_module,
"source_data": source_data,
"false_positive": false_positive,
"lastseen": lastseen
})
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.json"
cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"
cherrypy.response.headers['Pragma'] = "no-cache"
return json.dumps(scaninfo)
def stopscan(self, id, cli=None):
dbh = SpiderFootDb(self.config)
scaninfo = dbh.scanInstanceGet(id)
scanstatus = scaninfo[5]
print(scanstatus)
if scaninfo is None:
if not cli:
return self.error("Invalid scan ID.")
else:
return json.dumps(["ERROR", "Invalid scan ID."])
if scanstatus == "ABORTED":
if not cli:
return self.error("The scan is already aborted.")
else:
return json.dumps(["ERROR", "Scan already aborted."])
if not scanstatus == "RUNNING":
if not cli:
return self.error("The running scan is currently in the state '" + \
scanstatus + "', please try again later or restart " + \
" SpiderFoot.")
else:
return json.dumps(
["ERROR", "Scan in an invalid state for stopping."])
dbh.scanInstanceSet(id, status="ABORT-REQUESTED")
if not cli:
raise cherrypy.HTTPRedirect("/")
else:
return json.dumps(["SUCCESS", ""])
def scanopts(self, id):
ret = dict()
dbh = SpiderFootDb(self.config)
ret['config'] = dbh.scanConfigGet(id)
ret['configdesc'] = dict()
for key in ret['config'].keys():
if ':' not in key:
ret['configdesc'][key] = self.config['__globaloptdescs__'][key]
else:
[ modName, modOpt ] = key.split(':')
if not modName in self.config['__modules__'].keys():
continue
if not modOpt in self.config['__modules__'][modName]['optdescs'].keys():
continue
ret['configdesc'][key] = self.config['__modules__'][modName]['optdescs'][modOpt]
sf = SpiderFoot(self.config)
meta = dbh.scanInstanceGet(id)
if meta[3] != 0:
started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(meta[3]))
else:
started = "Not yet"
if meta[4] != 0:
finished = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(meta[4]))
else:
finished = "Not yet"
ret['meta'] = [meta[0], meta[1], meta[2], started, finished, meta[5]]
return json.dumps(ret)
def stopscanmulti(self, ids):
dbh = SpiderFootDb(self.config)
error = list()
for id in ids.split(","):
errState = False
scaninfo = dbh.scanInstanceGet(id)
if scaninfo is None:
return self.error("Invalid scan ID specified.")
scanname = str(scaninfo[0])
scanstatus = scaninfo[5]
if scanstatus == "FINISHED":
error.append("Scan '" + scanname + "' is in a finished state. <a href='/scandelete?id=" + \
id + "&confirm=1'>Maybe you want to delete it instead?</a>")
errState = True
if not errState and scanstatus == "ABORTED":
error.append("Scan '" + scanname + "' is already aborted.")
errState = True
if not errState:
#set the scanstatus in the db to "ABORT-REQUESTED"
dbh.scanInstanceSet(id, status="ABORT-REQUESTED")
raise cherrypy.HTTPRedirect("/")
def stopscan(self, id):
global globalScanStatus
dbh = SpiderFootDb(self.config)
scaninfo = dbh.scanInstanceGet(id)
if scaninfo is None:
return self.error("Invalid scan ID.")
if globalScanStatus.getStatus(id) is None:
return self.error("That scan is not actually running. A data consistency " + \
"error for this scan probably exists. <a href='/scandelete?id=" + \
id + "&confirm=1'>Click here to delete it.</a>")
if globalScanStatus.getStatus(id) == "ABORTED":
return self.error("The scan is already aborted.")
if not globalScanStatus.getStatus(id) == "RUNNING":
return self.error("The running scan is currently in the state '" + \
globalScanStatus.getStatus(id) + "', please try again later or restart " + \
" SpiderFoot.")
globalScanStatus.setStatus(id, "ABORT-REQUESTED")
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(pageid='SCANLIST',
stoppedscan=True,
docroot=self.docroot,
errors=list())
def stopscanmulti(self, ids):
#Removing All globalScanStatus Refs
dbh = SpiderFootDb(self.config)
error = list()
for id in ids.split(","):
errState = False
scaninfo = dbh.scanInstanceGet(id)
if scaninfo is None:
return self.error("Invalid scan ID specified.")
scanname = str(scaninfo[0])
scanstatus = scaninfo[5]
if scanstatus == "FINISHED":
error.append("Scan '" + scanname + "' is in a finished state. <a href='/scandelete?id=" + \
id + "&confirm=1'>Maybe you want to delete it instead?</a>")
errState = True
if not errState and scanstatus == "ABORTED":
error.append("Scan '" + scanname + "' is already aborted.")
errState = True
if not errState:
#set the scanstatus in the db to "ABORT-REQUESTED"
dbh.scanInstanceSet(id, status="ABORT-REQUESTED")
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(pageid='SCANLIST',
stoppedscan=True,
errors=error,
docroot=self.docroot)
def scandelete(self, id, confirm=None, raw=False):
dbh = SpiderFootDb(self.config)
res = dbh.scanInstanceGet(id)
if res is None:
if not raw:
return self.error("Scan ID not found.")
else:
return json.dumps(["ERROR", "Scan ID not found."])
if confirm is not None:
dbh.scanInstanceDelete(id)
if not raw:
raise cherrypy.HTTPRedirect("/")
else:
return json.dumps(["SUCCESS", ""])
else:
templ = Template(filename='dyn/scandelete.tmpl',
lookup=self.lookup)
return templ.render(id=id,
name=unicode(res[0], 'utf-8',
errors='replace'),
names=list(),
ids=list(),
pageid="SCANLIST",
docroot=self.docroot)
def scanopts(self, id):
ret = dict()
dbh = SpiderFootDb(self.config)
ret['config'] = dbh.scanConfigGet(id)
ret['configdesc'] = dict()
for key in ret['config'].keys():
if ':' not in key:
ret['configdesc'][key] = self.config['__globaloptdescs__'][key]
else:
[modName, modOpt] = key.split(':')
if modName not in self.config['__modules__'].keys():
continue
if modOpt not in self.config['__modules__'][modName]['optdescs'].keys():
continue
ret['configdesc'][key] = self.config['__modules__'][modName]['optdescs'][modOpt]
sf = SpiderFoot(self.config)
meta = dbh.scanInstanceGet(id)
if meta[3] != 0:
started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(meta[3]))
else:
started = "Not yet"
if meta[4] != 0:
finished = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(meta[4]))
else:
finished = "Not yet"
ret['meta'] = [meta[0], meta[1], meta[2], started, finished, meta[5]]
return json.dumps(ret)
def stopscan(self, id, cli=None):
dbh = SpiderFootDb(self.config)
scaninfo = dbh.scanInstanceGet(id)
scanstatus = scaninfo[5]
print(scanstatus)
if scaninfo is None:
if not cli:
return self.error("Invalid scan ID.")
else:
return json.dumps(["ERROR", "Invalid scan ID."])
if scanstatus == "ABORTED":
if not cli:
return self.error("The scan is already aborted.")
else:
return json.dumps(["ERROR", "Scan already aborted."])
if not scanstatus == "RUNNING":
if not cli:
return self.error("The running scan is currently in the state '" + \
scanstatus + "', please try again later or restart " + \
" SpiderFoot.")
else:
return json.dumps(
["ERROR", "Scan in an invalid state for stopping."])
dbh.scanInstanceSet(id, status="ABORT-REQUESTED")
if not cli:
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(pageid='SCANLIST',
stoppedscan=True,
docroot=self.docroot,
errors=list())
else:
return json.dumps(["SUCCESS", ""])
def resultsetfp(self, id, resultids, fp):
dbh = SpiderFootDb(self.config)
if fp not in ["0", "1"]:
return json.dumps(["ERROR", "No FP flag set or not set correctly."])
ids = json.loads(resultids)
if not ids:
return json.dumps(["ERROR", "No IDs supplied."])
# Cannot set FPs if a scan is not completed
status = dbh.scanInstanceGet(id)
if status[5] not in [ "ABORTED", "FINISHED", "ERROR-FAILED" ]:
return json.dumps(["WARNING", "Scan must be in a finished state when " + \
"setting False Positives."])
# Make sure the user doesn't set something as non-FP when the
# parent is set as an FP.
if fp == "0":
data = dbh.scanElementSourcesDirect(id, ids)
for row in data:
if str(row[14]) == "1":
return json.dumps(["WARNING",
"You cannot unset an element as False Positive " + \
"if a parent element is still False Positive."]);
# Set all the children as FPs too.. it's only logical afterall, right?
childs = dbh.scanElementChildrenAll(id, ids)
allIds = ids + childs
ret = dbh.scanResultsUpdateFP(id, allIds, fp)
if not ret:
return json.dumps(["ERROR", "Exception encountered."])
else:
return json.dumps(["SUCCESS", ""])
def stopscanmulti(self, ids):
global globalScanStatus # running scans
dbh = SpiderFootDb(self.config)
error = list()
for id in ids.split(","):
errState = False
scaninfo = dbh.scanInstanceGet(id)
if globalScanStatus.getStatus(id) == "FINISHED" or scaninfo[5] == "FINISHED":
error.append("Scan '" + scaninfo[0] + "' is in a finished state. <a href='/scandelete?id=" + \
id + "&confirm=1'>Maybe you want to delete it instead?</a>")
errState = True
if not errState and (globalScanStatus.getStatus(id) == "ABORTED" or scaninfo[5] == "ABORTED"):
error.append("Scan '" + scaninfo[0] + "' is already aborted.")
errState = True
if not errState and globalScanStatus.getStatus(id) is None:
error.append("Scan '" + scaninfo[0] + "' is not actually running. A data consistency " + \
"error for this scan probably exists. <a href='/scandelete?id=" + \
id + "&confirm=1'>Click here to delete it.</a>")
errState = True
if not errState:
globalScanStatus.setStatus(id, "ABORT-REQUESTED")
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(pageid='SCANLIST', stoppedscan=True,
errors=error, docroot=self.docroot)
def clonescan(self, id):
sf = SpiderFoot(self.config)
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
info = dbh.scanInstanceGet(id)
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if scanname == "" or scantarget == "" or len(scanconfig) == 0:
return self.error("Something went wrong internally.")
targetType = sf.targetType(scantarget)
if targetType == None:
# It must be a name, so wrap quotes around it
scantarget = """ + scantarget + """
modlist = scanconfig['_modulesenabled'].split(',')
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(pageid='NEWSCAN',
types=types,
docroot=self.docroot,
modules=self.config['__modules__'],
selectedmods=modlist,
scanname=str(scanname),
scantarget=str(scantarget))
def scaninfo(self, id):
dbh = SpiderFootDb(self.config)
res = dbh.scanInstanceGet(id)
if res == None:
return self.error("Scan ID not found.")
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
return templ.render(id=id, name=res[0], status=res[5], pageid='SCANLIST')
def scaninfo(self, id):
dbh = SpiderFootDb(self.config)
res = dbh.scanInstanceGet(id)
if res is None:
return self.error("Scan ID not found.")
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup, disable_unicode=True, input_encoding='utf-8')
return templ.render(id=id, name=cgi.escape(res[0]), status=res[5], docroot=self.docroot,
pageid="SCANLIST")
def scaninfo(self, id):
dbh = SpiderFootDb(self.config)
res = dbh.scanInstanceGet(id)
if res is None:
return self.error("Scan ID not found.")
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup, disable_unicode=True, input_encoding='utf-8')
return templ.render(id=id, name=cgi.escape(res[0]), status=res[5], docroot=self.docroot,
pageid="SCANLIST")
def scanstatus(self, id):
dbh = SpiderFootDb(self.config)
data = dbh.scanInstanceGet(id)
created = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[2]))
started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[3]))
ended = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[4]))
retdata = [data[0], data[1], created, started, ended, data[5]]
return json.dumps(retdata)
def scanstatus(self, id):
dbh = SpiderFootDb(self.config)
data = dbh.scanInstanceGet(id)
created = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[2]))
started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[3]))
ended = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[4]))
retdata = [data[0], data[1], created, started, ended, data[5]]
return json.dumps(retdata)
def rerunscanmulti(self, ids):
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
for id in ids.split(","):
info = dbh.scanInstanceGet(id)
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if len(scanconfig) == 0:
return self.error("Something went wrong internally.")
modlist = scanconfig['_modulesenabled'].split(',')
if "sfp__stor_stdout" in modlist:
modlist.remove("sfp__stor_stdout")
targetType = sf.targetType(scantarget)
if targetType == None:
# Should never be triggered for a re-run scan..
return self.error("Invalid target type. Could not recognize it as " + \
"a human name, IP address, IP subnet, ASN, domain name or host name.")
# Start running a new scan
newId = sf.genScanInstanceGUID(scanname)
p = mp.Process(target=SpiderFootScanner,
args=(scanname, scantarget.lower(), targetType,
newId, modlist, cfg, modopts))
p.start()
# Wait until the scan has initialized
while dbh.scanInstanceGet(newId) == None:
print("[info] Waiting for the scan to initialize...")
time.sleep(1)
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(rerunscans=True,
docroot=self.docroot,
pageid="SCANLIST")
def test_scanInstanceGet_argument_instanceId_of_invalid_type_should_raise_TypeError(self):
"""
Test scanInstanceGet(self, instanceId)
"""
sfdb = SpiderFootDb(self.default_options, False)
invalid_types = [None, list(), dict(), int()]
for invalid_type in invalid_types:
with self.subTest(invalid_type=invalid_type):
with self.assertRaises(TypeError) as cm:
scan_instance_get = sfdb.scanInstanceGet(invalid_type)
def test_scan_instance_get_should_return_a_list(self):
"""
Test scanInstanceGet(self, instanceId)
"""
sfdb = SpiderFootDb(self.default_options, False)
scan_instance = 'example scan instance'
sfdb.scanInstanceSet(scan_instance, None, None, None)
scan_instance_get = sfdb.scanInstanceGet(scan_instance)
self.assertIsInstance(scan_instance_get, list)
def scandelete(self, id, confirm=None):
dbh = SpiderFootDb(self.config)
res = dbh.scanInstanceGet(id)
if res == None:
return self.error("Scan ID not found.")
if confirm != None:
dbh.scanInstanceDelete(id)
raise cherrypy.HTTPRedirect("/")
else:
templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup)
return templ.render(id=id, name=res[0])
def scanviz(self, id, gexf="0"):
dbh = SpiderFootDb(self.config)
sf = SpiderFoot(self.config)
data = dbh.scanResultEvent(id, filterFp=True)
scan = dbh.scanInstanceGet(id)
root = scan[1]
if gexf != "0":
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.gexf"
cherrypy.response.headers['Content-Type'] = "application/gexf"
cherrypy.response.headers['Pragma'] = "no-cache"
return sf.buildGraphGexf([root], "SpiderFoot Export", data)
else:
return sf.buildGraphJson([root], data)
def scandelete(self, id, confirm=None):
dbh = SpiderFootDb(self.config)
res = dbh.scanInstanceGet(id)
if res is None:
return self.error("Scan ID not found.")
if confirm is not None:
dbh.scanInstanceDelete(id)
raise cherrypy.HTTPRedirect("/")
else:
templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup)
return templ.render(id=id, name=res[0], names=list(), ids=list(),
pageid="SCANLIST", docroot=self.docroot)
def scanviz(self, id, gexf="0"):
types = list()
dbh = SpiderFootDb(self.config)
sf = SpiderFoot(self.config)
data = dbh.scanResultEvent(id, filterFp=True)
scan = dbh.scanInstanceGet(id)
root = scan[1]
if gexf != "0":
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.gexf"
cherrypy.response.headers['Content-Type'] = "application/gexf"
cherrypy.response.headers['Pragma'] = "no-cache"
return sf.buildGraphGexf([root], "SpiderFoot Export", data)
else:
return sf.buildGraphJson([root], data)
def scanexportjsonmulti(self, ids):
dbh = SpiderFootDb(self.config)
scaninfo = list()
scan_name = ""
for id in ids.split(','):
scan = dbh.scanInstanceGet(id)
if scan is None:
continue
scan_name = scan[0]
for row in dbh.scanResultEvent(id):
lastseen = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0]))
event_data = str(row[1]).replace("<SFURL>",
"").replace("</SFURL>", "")
source_data = str(row[2])
source_module = str(row[3])
event_type = row[4]
false_positive = row[13]
if event_type == "ROOT":
continue
scaninfo.append({
"data": event_data,
"event_type": event_type,
"module": source_module,
"source_data": source_data,
"false_positive": false_positive,
"last_seen": lastseen,
"scan_name": scan_name,
"scan_target": scan[1]
})
if len(ids.split(',')) > 1 or scan_name == "":
fname = "SpiderFoot.json"
else:
fname = scan_name + "-SpiderFoot.json"
cherrypy.response.headers[
'Content-Disposition'] = "attachment; filename=" + fname
cherrypy.response.headers[
'Content-Type'] = "application/json; charset=utf-8"
cherrypy.response.headers['Pragma'] = "no-cache"
return json.dumps(scaninfo).encode("utf-8")
def scanvizmulti(self, ids, gexf="1"):
dbh = SpiderFootDb(self.config)
sf = SpiderFoot(self.config)
data = list()
roots = list()
for id in ids.split(','):
data = data + dbh.scanResultEvent(id, filterFp=True)
roots.append(dbh.scanInstanceGet(id)[1])
if gexf != "0":
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.gexf"
cherrypy.response.headers['Content-Type'] = "application/gexf"
cherrypy.response.headers['Pragma'] = "no-cache"
return sf.buildGraphGexf(roots, "SpiderFoot Export", data)
else:
# Not implemented yet
return None
def clonescan(self, id):
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
info = dbh.scanInstanceGet(id)
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if scanname == "" or scantarget == "" or len(scanconfig) == 0:
return self.error("Something went wrong internally.")
modlist = scanconfig['_modulesenabled'].split(',')
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(pageid='NEWSCAN', types=types, docroot=self.docroot,
modules=self.config['__modules__'], selectedmods=modlist,
scanname=scanname, scantarget=scantarget)
def scanvizmulti(self, ids, gexf="1"):
types = list()
dbh = SpiderFootDb(self.config)
sf = SpiderFoot(self.config)
data = list()
roots = list()
for id in ids.split(','):
data = data + dbh.scanResultEvent(id, filterFp=True)
roots.append(dbh.scanInstanceGet(id)[1])
if gexf != "0":
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.gexf"
cherrypy.response.headers['Content-Type'] = "application/gexf"
cherrypy.response.headers['Pragma'] = "no-cache"
return sf.buildGraphGexf(roots, "SpiderFoot Export", data)
else:
# Not implemented yet
return None
def scandelete(self, id, confirm=None, raw=False):
dbh = SpiderFootDb(self.config)
res = dbh.scanInstanceGet(id)
if res is None:
if not raw:
return self.error("Scan ID not found.")
else:
return json.dumps(["ERROR", "Scan ID not found."])
if confirm is not None:
dbh.scanInstanceDelete(id)
if not raw:
raise cherrypy.HTTPRedirect("/")
else:
return json.dumps(["SUCCESS", ""])
else:
templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup)
return templ.render(id=id, name=unicode(res[0], 'utf-8', errors='replace'),
names=list(), ids=list(),
pageid="SCANLIST", docroot=self.docroot)
def rerunscan(self, id):
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
info = dbh.scanInstanceGet(id)
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if len(scanconfig) == 0:
return self.error("Something went wrong internally.")
modlist = scanconfig['_modulesenabled'].split(',')
targetType = sf.targetType(scantarget)
if targetType == None:
# It must then be a name, as a re-run scan should always have a clean
# target.
targetType = "HUMAN_NAME"
if targetType != "HUMAN_NAME":
scantarget = scantarget.lower()
# Start running a new scan
newId = sf.genScanInstanceGUID(scanname)
t = SpiderFootScanner(scanname, scantarget, targetType, newId,
modlist, cfg, modopts)
t.start()
# Wait until the scan has initialized
while globalScanStatus.getStatus(newId) == None:
print "[info] Waiting for the scan to initialize..."
time.sleep(1)
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
return templ.render(id=newId, name=unicode(scanname, 'utf-8', errors='replace'), docroot=self.docroot,
status=globalScanStatus.getStatus(newId), pageid="SCANLIST")
def scaneventresultexportmulti(self, ids, dialect="excel"):
dbh = SpiderFootDb(self.config)
scaninfo = dict()
data = list()
for id in ids.split(','):
scaninfo[id] = dbh.scanInstanceGet(id)
data = data + dbh.scanResultEvent(id)
fileobj = StringIO()
parser = csv.writer(fileobj, dialect=dialect)
parser.writerow(["Scan Name", "Updated", "Type", "Module", "Source", "Data"])
for row in data:
if row[4] == "ROOT":
continue
lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0]))
datafield = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "")
parser.writerow([scaninfo[row[12]][0], lastseen, str(row[4]), str(row[3]), str(row[2]), datafield])
cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.csv"
cherrypy.response.headers['Content-Type'] = "application/csv"
cherrypy.response.headers['Pragma'] = "no-cache"
return fileobj.getvalue()
def scandeletemulti(self, ids, confirm=None):
dbh = SpiderFootDb(self.config)
names = list()
for id in ids.split(','):
res = dbh.scanInstanceGet(id)
names.append(res[0])
if res is None:
return self.error("Scan ID not found (" + id + ").")
if res[5] in [ "RUNNING", "STARTING", "STARTED" ]:
return self.error("You cannot delete running scans.")
if confirm is not None:
for id in ids.split(','):
dbh.scanInstanceDelete(id)
raise cherrypy.HTTPRedirect("/")
else:
templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup)
return templ.render(id=None, name=None, ids=ids.split(','), names=names,
pageid="SCANLIST", docroot=self.docroot)
def stopscan(self, id, cli=None):
global globalScanStatus
dbh = SpiderFootDb(self.config)
scaninfo = dbh.scanInstanceGet(id)
if scaninfo is None:
if not cli:
return self.error("Invalid scan ID.")
else:
return json.dumps(["ERROR", "Invalid scan ID."])
if globalScanStatus.getStatus(id) is None:
if not cli:
return self.error("That scan is not actually running. A data consistency " + \
"error for this scan probably exists. <a href='/scandelete?id=" + \
id + "&confirm=1'>Click here to delete it.</a>")
else:
return json.dumps(["ERROR", "Scan doesn't appear to be running."])
if globalScanStatus.getStatus(id) == "ABORTED":
if not cli:
return self.error("The scan is already aborted.")
else:
return json.dumps(["ERROR", "Scan already aborted."])
if not globalScanStatus.getStatus(id) == "RUNNING":
if not cli:
return self.error("The running scan is currently in the state '" + \
globalScanStatus.getStatus(id) + "', please try again later or restart " + \
" SpiderFoot.")
else:
return json.dumps(["ERROR", "Scan in an invalid state for stopping."])
globalScanStatus.setStatus(id, "ABORT-REQUESTED")
if not cli:
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(pageid='SCANLIST', stoppedscan=True, docroot=self.docroot, errors=list())
else:
return json.dumps(["SUCCESS", ""])
def rerunscanmulti(self, ids):
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
for id in ids.split(","):
info = dbh.scanInstanceGet(id)
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if len(scanconfig) == 0:
return self.error("Something went wrong internally.")
modlist = scanconfig['_modulesenabled'].split(',')
targetType = sf.targetType(scantarget)
if targetType == None:
# Should never be triggered for a re-run scan..
return self.error("Invalid target type. Could not recognize it as " + \
"a human name, IP address, IP subnet, ASN, domain name or host name.")
# Start running a new scan
newId = sf.genScanInstanceGUID(scanname)
t = SpiderFootScanner(unicode(scanname, 'utf-8', errors='replace'),
unicode(scantarget, 'utf-8', errors='replace').lower(),
targetType, newId, modlist, cfg, modopts)
t.start()
# Wait until the scan has initialized
while globalScanStatus.getStatus(newId) == None:
print "[info] Waiting for the scan to initialize..."
time.sleep(1)
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(rerunscans=True, docroot=self.docroot, pageid="SCANLIST")
if args.debug:
cfg['_debug'] = True
else:
cfg['_debug'] = False
# If strict mode is enabled, filter the output from modules.
if args.x and args.t:
cfg['__outputfilter'] = args.t.split(",")
t = SpiderFootScanner(target, target, targetType, scanId,
modlist, cfg, dict())
t.daemon = True
t.start()
while True:
info = dbh.scanInstanceGet(scanId)
if not info:
time.sleep(1)
continue
if info[5] in [ "ERROR-FAILED", "ABORT-REQUESTED", "ABORTED", "FINISHED" ]:
if not args.q:
print "[*] Scan completed with status " + info[5]
sys.exit(0)
time.sleep(1)
sys.exit(0)
# Start the web server so you can start looking at results
print "Starting web server at http://" + sfConfig['__webaddr'] + \
":" + str(sfConfig['__webport']) + sfConfig['__docroot'] + " ..."
