def __init__(self, generated_root):
from mako.lookup import TemplateLookup
self.generated_root = generated_root
self.template_lookup = TemplateLookup(directories=['templates'])
from mako.lookup import TemplateLookup
import furl
import requests
from framework.exceptions import PermissionsError
from framework.mongo import StoredObject
from framework.routing import process_rules
from framework.guid.model import GuidStoredObject
from website import settings
from website.addons.base import exceptions
from website.addons.base import serializer
from website.project.model import Node
lookup = TemplateLookup(directories=[settings.TEMPLATES_PATH])
STATUS_EXCEPTIONS = {
410: exceptions.FileDeletedError,
404: exceptions.FileDoesntExistError
}
def _is_image(filename):
mtype, _ = mimetypes.guess_type(filename)
return mtype and mtype.startswith('image')
class AddonConfig(object):
def __init__(self,
short_name,
def __init__(self, view_search_folder):
self.lookup_o = TemplateLookup(directories=view_search_folder,
input_encoding='utf-8')
class SpiderFootWebUi:
lookup = TemplateLookup(directories=[''], output_encoding='utf-8')
defaultConfig = dict()
config = dict()
token = None
docroot = ''
def __init__(self, config):
self.defaultConfig = deepcopy(config)
dbh = SpiderFootDb(self.defaultConfig)
# 'config' supplied will be the defaults, let's supplement them
# now with any configuration which may have previously been
# saved.
sf = SpiderFoot(self.defaultConfig)
self.config = sf.configUnserialize(dbh.configGet(), self.defaultConfig)
if self.config['__webaddr'] == "0.0.0.0":
addr = "<IP of this host>"
else:
addr = self.config['__webaddr']
self.docroot = self.config['__docroot'].rstrip('/')
print ""
print ""
print "*************************************************************"
print " Use SpiderFoot by starting your web browser of choice and "
print " browse to http://" + addr + ":" + str(
self.config['__webport']) + self.docroot
print "*************************************************************"
print ""
print ""
# Sanitize user input
def cleanUserInput(self, inputList):
ret = list()
for item in inputList:
c = cgi.escape(item, True)
c = c.replace('\'', '"')
# We don't actually want & translated to &
c = c.replace("&", "&")
ret.append(c)
return ret
def searchBase(self, id=None, eventType=None, value=None):
regex = ""
if [id, eventType, value].count('') == 2 or \
[id, eventType, value].count(None) == 2:
return None
if value.startswith("/") and value.endswith("/"):
regex = value[1:len(value) - 1]
value = ""
value = value.replace('*', '%')
if value in [None, ""] and regex in [None, ""]:
value = "%"
regex = ""
dbh = SpiderFootDb(self.config)
criteria = {
'scan_id': None if id == '' else id,
'type': None if eventType == '' else eventType,
'value': None if value == '' else value,
'regex': None if regex == '' else regex
}
data = dbh.search(criteria)
retdata = []
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0]))
escapeddata = cgi.escape(row[1])
escapedsrc = cgi.escape(row[2])
retdata.append([
lastseen, escapeddata, escapedsrc, row[3], row[5], row[6],
row[7], row[8], row[10], row[11], row[4], row[13], row[14]
])
return retdata
#
# USER INTERFACE PAGES
#
# Get result data in CSV format
def scaneventresultexport(self, id, type, dialect="excel"):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultEvent(id, type)
fileobj = StringIO()
parser = csv.writer(fileobj, dialect=dialect)
parser.writerow(["Updated", "Type", "Module", "Source", "F/P", "Data"])
for row in data:
if row[4] == "ROOT":
continue
lastseen = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0]))
datafield = str(row[1]).replace("<SFURL>",
"").replace("</SFURL>", "")
parser.writerow([
lastseen,
str(row[4]),
str(row[3]),
str(row[2]), row[13], datafield
])
cherrypy.response.headers[
'Content-Disposition'] = "attachment; filename=SpiderFoot.csv"
cherrypy.response.headers['Content-Type'] = "application/csv"
cherrypy.response.headers['Pragma'] = "no-cache"
return fileobj.getvalue()
scaneventresultexport.exposed = True
# Get result data in CSV format for multiple scans
def scaneventresultexportmulti(self, ids, dialect="excel"):
dbh = SpiderFootDb(self.config)
scaninfo = dict()
data = list()
for id in ids.split(','):
scaninfo[id] = dbh.scanInstanceGet(id)
data = data + dbh.scanResultEvent(id)
fileobj = StringIO()
parser = csv.writer(fileobj, dialect=dialect)
parser.writerow([
"Scan Name", "Updated", "Type", "Module", "Source", "F/P", "Data"
])
for row in data:
if row[4] == "ROOT":
continue
lastseen = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0]))
datafield = str(row[1]).replace("<SFURL>",
"").replace("</SFURL>", "")
parser.writerow([
scaninfo[row[12]][0], lastseen,
str(row[4]),
str(row[3]),
str(row[2]), row[13], datafield
])
cherrypy.response.headers[
'Content-Disposition'] = "attachment; filename=SpiderFoot.csv"
cherrypy.response.headers['Content-Type'] = "application/csv"
cherrypy.response.headers['Pragma'] = "no-cache"
return fileobj.getvalue()
scaneventresultexportmulti.exposed = True
# Get search result data in CSV format
def scansearchresultexport(self,
id,
eventType=None,
value=None,
dialect="excel"):
data = self.searchBase(id, eventType, value)
fileobj = StringIO()
parser = csv.writer(fileobj, dialect=dialect)
parser.writerow(["Updated", "Type", "Module", "Source", "F/P", "Data"])
for row in data:
if row[10] == "ROOT":
continue
datafield = str(row[1]).replace("<SFURL>",
"").replace("</SFURL>", "")
parser.writerow([
row[0],
str(row[10]),
str(row[3]),
str(row[2]), row[13], datafield
])
cherrypy.response.headers[
'Content-Disposition'] = "attachment; filename=SpiderFoot.csv"
cherrypy.response.headers['Content-Type'] = "application/csv"
cherrypy.response.headers['Pragma'] = "no-cache"
return fileobj.getvalue()
scansearchresultexport.exposed = True
# Export entities from scan results for visualising
def scanviz(self, id, gexf="0"):
types = list()
dbh = SpiderFootDb(self.config)
sf = SpiderFoot(self.config)
data = dbh.scanResultEvent(id, filterFp=True)
scan = dbh.scanInstanceGet(id)
root = scan[1]
if gexf != "0":
cherrypy.response.headers[
'Content-Disposition'] = "attachment; filename=SpiderFoot.gexf"
cherrypy.response.headers['Content-Type'] = "application/gexf"
cherrypy.response.headers['Pragma'] = "no-cache"
return sf.buildGraphGexf([root], "SpiderFoot Export", data)
else:
return sf.buildGraphJson([root], data)
scanviz.exposed = True
# Export entities results from multiple scans in GEXF format
def scanvizmulti(self, ids, gexf="1"):
types = list()
dbh = SpiderFootDb(self.config)
sf = SpiderFoot(self.config)
data = list()
roots = list()
for id in ids.split(','):
data = data + dbh.scanResultEvent(id, filterFp=True)
roots.append(dbh.scanInstanceGet(id)[1])
if gexf != "0":
cherrypy.response.headers[
'Content-Disposition'] = "attachment; filename=SpiderFoot.gexf"
cherrypy.response.headers['Content-Type'] = "application/gexf"
cherrypy.response.headers['Pragma'] = "no-cache"
return sf.buildGraphGexf(roots, "SpiderFoot Export", data)
else:
# Not implemented yet
return None
scanvizmulti.exposed = True
# Configuration used for a scan
def scanopts(self, id):
ret = dict()
dbh = SpiderFootDb(self.config)
ret['config'] = dbh.scanConfigGet(id)
ret['configdesc'] = dict()
for key in ret['config'].keys():
if ':' not in key:
ret['configdesc'][key] = self.config['__globaloptdescs__'][key]
else:
[modName, modOpt] = key.split(':')
if modName not in self.config['__modules__'].keys():
continue
if modOpt not in self.config['__modules__'][modName][
'optdescs'].keys():
continue
ret['configdesc'][key] = self.config['__modules__'][modName][
'optdescs'][modOpt]
sf = SpiderFoot(self.config)
meta = dbh.scanInstanceGet(id)
if not meta:
return json.dumps([])
if meta[3] != 0:
started = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(meta[3]))
else:
started = "Not yet"
if meta[4] != 0:
finished = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(meta[4]))
else:
finished = "Not yet"
ret['meta'] = [meta[0], meta[1], meta[2], started, finished, meta[5]]
return json.dumps(ret)
scanopts.exposed = True
def rerunscan(self, id):
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
info = dbh.scanInstanceGet(id)
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if len(scanconfig) == 0:
return self.error("Something went wrong internally.")
modlist = scanconfig['_modulesenabled'].split(',')
targetType = sf.targetType(scantarget)
if targetType == None:
# Should never be triggered for a re-run scan..
return self.error("Invalid target type. Could not recognize it as " + \
"an IP address, IP subnet, domain name or host name.")
# Start running a new scan
newId = sf.genScanInstanceGUID(scanname)
t = SpiderFootScanner(scanname, scantarget.lower(), targetType, newId,
modlist, cfg, modopts)
t.start()
# Wait until the scan has initialized
while globalScanStatus.getStatus(newId) == None:
print "[info] Waiting for the scan to initialize..."
time.sleep(1)
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
return templ.render(id=newId,
name=scanname,
docroot=self.docroot,
status=globalScanStatus.getStatus(newId),
pageid="SCANLIST")
rerunscan.exposed = True
def rerunscanmulti(self, ids):
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
for id in ids.split(","):
info = dbh.scanInstanceGet(id)
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if len(scanconfig) == 0:
return self.error("Something went wrong internally.")
modlist = scanconfig['_modulesenabled'].split(',')
targetType = sf.targetType(scantarget)
if targetType == None:
# Should never be triggered for a re-run scan..
return self.error("Invalid target type. Could not recognize it as " + \
"an IP address, IP subnet, domain name or host name.")
# Start running a new scan
newId = sf.genScanInstanceGUID(scanname)
t = SpiderFootScanner(scanname, scantarget.lower(), targetType,
newId, modlist, cfg, modopts)
t.start()
# Wait until the scan has initialized
while globalScanStatus.getStatus(newId) == None:
print "[info] Waiting for the scan to initialize..."
time.sleep(1)
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(rerunscans=True,
docroot=self.docroot,
pageid="SCANLIST")
rerunscanmulti.exposed = True
# Configure a new scan
def newscan(self):
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(pageid='NEWSCAN',
types=types,
docroot=self.docroot,
modules=self.config['__modules__'],
scanname="",
selectedmods="",
scantarget="")
newscan.exposed = True
# Clone an existing scan (pre-selected options in the newscan page)
def clonescan(self, id):
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
info = dbh.scanInstanceGet(id)
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if scanname == "" or scantarget == "" or len(scanconfig) == 0:
return self.error("Something went wrong internally.")
modlist = scanconfig['_modulesenabled'].split(',')
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(pageid='NEWSCAN',
types=types,
docroot=self.docroot,
modules=self.config['__modules__'],
selectedmods=modlist,
scanname=scanname,
scantarget=scantarget)
clonescan.exposed = True
# Main page listing scans available
def index(self):
# Look for referenced templates in the current directory only
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(pageid='SCANLIST', docroot=self.docroot)
index.exposed = True
# Information about a selected scan
def scaninfo(self, id):
dbh = SpiderFootDb(self.config)
res = dbh.scanInstanceGet(id)
if res is None:
return self.error("Scan ID not found.")
scanName = SpiderFoot.znEncode(res[0])
target = res[1]
templ = Template(filename='dyn/scaninfo.tmpl',
lookup=self.lookup,
input_encoding='utf-8')
# templ = Template(filename='dyn/scaninfo.tmpl', default_filters=['decode.utf8'], input_encoding='utf-8', output_encoding='utf-8')
return templ.render(id=id,
name=target,
status=res[5],
docroot=self.docroot,
pageid="SCANLIST")
scaninfo.exposed = True
# Settings
def opts(self):
templ = Template(filename='dyn/opts.tmpl', lookup=self.lookup)
self.token = random.randint(0, 99999999)
return templ.render(opts=self.config,
pageid='SETTINGS',
token=self.token,
docroot=self.docroot)
opts.exposed = True
# Settings
def optsraw(self):
ret = dict()
self.token = random.randint(0, 99999999)
for opt in self.config:
if opt.startswith('__'):
if opt == '__modules__':
for mod in sorted(self.config['__modules__'].keys()):
for mo in sorted(self.config['__modules__'][mod]
['opts'].keys()):
if mo.startswith("_"):
continue
ret["module." + mod + "." + mo] = self.config[
'__modules__'][mod]['opts'][mo]
continue
ret["global." + opt] = self.config[opt]
return json.dumps(['SUCCESS', {'token': self.token, 'data': ret}])
optsraw.exposed = True
# Generic error, but not exposed as not called directly
def error(self, message):
templ = Template(filename='dyn/error.tmpl', lookup=self.lookup)
return templ.render(message=message, docroot=self.docroot)
# Delete a scan
def scandelete(self, id, confirm=None, raw=False):
dbh = SpiderFootDb(self.config)
res = dbh.scanInstanceGet(id)
if res is None:
if not raw:
return self.error("Scan ID not found.")
else:
return json.dumps(["ERROR", "Scan ID not found."])
if confirm is not None:
dbh.scanInstanceDelete(id)
if not raw:
raise cherrypy.HTTPRedirect("/")
else:
return json.dumps(["SUCCESS", ""])
else:
templ = Template(filename='dyn/scandelete.tmpl',
lookup=self.lookup)
return templ.render(id=id,
name=res[0],
names=list(),
ids=list(),
pageid="SCANLIST",
docroot=self.docroot)
scandelete.exposed = True
# Delete a scan
def scandeletemulti(self, ids, confirm=None):
dbh = SpiderFootDb(self.config)
names = list()
for id in ids.split(','):
res = dbh.scanInstanceGet(id)
names.append(res[0])
if res is None:
return self.error("Scan ID not found (" + id + ").")
if res[5] in ["RUNNING", "STARTING", "STARTED"]:
return self.error("You cannot delete running scans.")
if confirm is not None:
for id in ids.split(','):
dbh.scanInstanceDelete(id)
raise cherrypy.HTTPRedirect("/")
else:
templ = Template(filename='dyn/scandelete.tmpl',
lookup=self.lookup)
return templ.render(id=None,
name=None,
ids=ids.split(','),
names=names,
pageid="SCANLIST",
docroot=self.docroot)
scandeletemulti.exposed = True
# Save settings, also used to completely reset them to default
def savesettings(self, allopts, token):
# if str(token) != str(self.token):
# return self.error("Invalid token (" + str(self.token) + ").")
try:
dbh = SpiderFootDb(self.config)
# Reset config to default
if allopts == "RESET":
dbh.configClear() # Clear it in the DB
self.config = deepcopy(self.defaultConfig) # Clear in memory
else:
useropts = json.loads(allopts)
cleanopts = dict()
for opt in useropts.keys():
cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0]
currentopts = deepcopy(self.config)
# Make a new config where the user options override
# the current system config.
sf = SpiderFoot(self.config)
self.config = sf.configUnserialize(cleanopts, currentopts)
dbh.configSet(sf.configSerialize(currentopts))
except Exception as e:
return self.error(
"Processing one or more of your inputs failed: " + str(e))
templ = Template(filename='dyn/opts.tmpl', lookup=self.lookup)
self.token = random.randint(0, 99999999)
return templ.render(opts=self.config,
pageid='SETTINGS',
updated=True,
docroot=self.docroot,
token=self.token)
savesettings.exposed = True
# Save settings, also used to completely reset them to default
def savesettingsraw(self, allopts, token):
if str(token) != str(self.token):
return json.dumps(
["ERROR", "Invalid token (" + str(self.token) + ")."])
try:
dbh = SpiderFootDb(self.config)
# Reset config to default
if allopts == "RESET":
dbh.configClear() # Clear it in the DB
self.config = deepcopy(self.defaultConfig) # Clear in memory
else:
useropts = json.loads(allopts)
cleanopts = dict()
for opt in useropts.keys():
cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0]
currentopts = deepcopy(self.config)
# Make a new config where the user options override
# the current system config.
sf = SpiderFoot(self.config)
self.config = sf.configUnserialize(cleanopts, currentopts)
dbh.configSet(sf.configSerialize(currentopts))
except Exception as e:
return json.dumps([
"ERROR",
"Processing one or more of your inputs failed: " + str(e)
])
return json.dumps(["SUCCESS", ""])
savesettingsraw.exposed = True
# Set a bunch of results (hashes) as false positive
def resultsetfp(self, id, resultids, fp):
dbh = SpiderFootDb(self.config)
if fp not in ["0", "1"]:
return json.dumps(
["ERROR", "No FP flag set or not set correctly."])
ids = json.loads(resultids)
if not ids:
return json.dumps(["ERROR", "No IDs supplied."])
# Cannot set FPs if a scan is not completed
status = dbh.scanInstanceGet(id)
if status[5] not in ["ABORTED", "FINISHED", "ERROR-FAILED"]:
return json.dumps(["WARNING", "Scan must be in a finished state when " + \
"setting False Positives."])
# Make sure the user doesn't set something as non-FP when the
# parent is set as an FP.
if fp == "0":
data = dbh.scanElementSourcesDirect(id, ids)
for row in data:
if str(row[14]) == "1":
return json.dumps(["WARNING",
"You cannot unset an element as False Positive " + \
"if a parent element is still False Positive."])
# Set all the children as FPs too.. it's only logical afterall, right?
childs = dbh.scanElementChildrenAll(id, ids)
allIds = ids + childs
ret = dbh.scanResultsUpdateFP(id, allIds, fp)
if not ret:
return json.dumps(["ERROR", "Exception encountered."])
else:
return json.dumps(["SUCCESS", ""])
resultsetfp.exposed = True
# For the CLI to fetch a list of event types.
def eventtypes(self):
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
ret = list()
for r in types:
ret.append([r[1], r[0]])
ret = sorted(ret, key=itemgetter(0))
return json.dumps(ret)
eventtypes.exposed = True
# For the CLI to fetch a list of modules.
def modules(self):
modinfo = self.config['__modules__'].keys()
modinfo.sort()
ret = list()
for m in modinfo:
if "__" in m:
continue
ret.append({
'name': m,
'descr': self.config['__modules__'][m]['descr']
})
return json.dumps(ret)
modules.exposed = True
# For the CLI to test connectivity to this server.
def ping(self):
return json.dumps(["SUCCESS", self.config['__version__']])
ping.exposed = True
# For the CLI to run queries against the database.
def query(self, query):
data = None
dbh = SpiderFootDb(self.config)
if not query.lower().startswith("select"):
return json.dumps([
"ERROR", "Non-SELECTs are unpredictable and not recommended."
])
try:
ret = dbh.dbh.execute(query)
data = ret.fetchall()
except BaseException as e:
return json.dumps(["ERROR", str(e)])
return json.dumps(data)
query.exposed = True
# Initiate a scan
def startscan(self,
scanname,
scantarget,
modulelist,
typelist,
usecase,
cli=None):
global globalScanStatus
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
types = dbh.eventTypes()
targetType = None
scanname = SpiderFoot.znEncode(scanname)
[scanname, scantarget] = self.cleanUserInput([scanname, scantarget])
if scanname == "" or scantarget == "":
if not cli:
return self.error("Form incomplete.")
else:
return json.dumps(["ERROR", "Incorrect usage."])
if typelist == "" and modulelist == "" and usecase == "":
if not cli:
return self.error("Form incomplete.")
else:
return json.dumps(["ERROR", "Incorrect usage."])
# User selected modules
if modulelist != "":
modlist = modulelist.replace('module_', '').split(',')
# User selected types
if len(modlist) == 0 and typelist != "":
typesx = typelist.replace('type_', '').split(',')
# 1. Find all modules that produce the requested types
modlist = sf.modulesProducing(typesx)
newmods = deepcopy(modlist)
newmodcpy = deepcopy(newmods)
# 2. For each type those modules consume, get modules producing
while len(newmodcpy) > 0:
for etype in sf.eventsToModules(newmodcpy):
xmods = sf.modulesProducing([etype])
for mod in xmods:
if mod not in modlist:
modlist.append(mod)
newmods.append(mod)
newmodcpy = deepcopy(newmods)
newmods = list()
# User selected a use case
if len(modlist) == 0 and usecase != "":
for mod in self.config['__modules__']:
if usecase == 'all' or usecase in self.config['__modules__'][
mod]['cats']:
modlist.append(mod)
# Add our mandatory storage module..
if "sfp__stor_db" not in modlist:
modlist.append("sfp__stor_db")
modlist.sort()
targetType = sf.targetType(scantarget)
if targetType is None:
if not cli:
return self.error("Invalid target type. Could not recognize it as " + \
"an IP address, IP subnet, domain name or host name.")
else:
return json.dumps(["ERROR", "Unrecognised target type."])
# Start running a new scan
scanId = sf.genScanInstanceGUID(scanname)
t = SpiderFootScanner(scanname, scantarget.lower(), targetType, scanId,
modlist, cfg, modopts)
t.start()
# Wait until the scan has initialized
while globalScanStatus.getStatus(scanId) is None:
print "[info] Waiting for the scan to initialize..."
time.sleep(1)
scanname = SpiderFoot.znDecode(scanname)
if not cli:
templ = Template(filename='dyn/scaninfo.tmpl',
lookup=self.lookup,
input_encoding='utf-8')
return templ.render(id=scanId,
name=scanname,
docroot=self.docroot,
status=globalScanStatus.getStatus(scanId),
pageid="SCANLIST")
else:
return json.dumps(["SUCCESS", scanId])
startscan.exposed = True
# Stop a scan (id variable is unnecessary for now given that only one simultaneous
# scan is permitted.)
def stopscanmulti(self, ids):
global globalScanStatus # running scans
dbh = SpiderFootDb(self.config)
error = list()
for id in ids.split(","):
errState = False
scaninfo = dbh.scanInstanceGet(id)
if scaninfo is None:
return self.error("Invalid scan ID specified.")
if globalScanStatus.getStatus(
id) == "FINISHED" or scaninfo[5] == "FINISHED":
error.append("Scan '" + scaninfo[0] + "' is in a finished state. <a href='/scandelete?id=" + \
id + "&confirm=1'>Maybe you want to delete it instead?</a>")
errState = True
if not errState and (globalScanStatus.getStatus(id) == "ABORTED"
or scaninfo[5] == "ABORTED"):
error.append("Scan '" + scaninfo[0] + "' is already aborted.")
errState = True
if not errState and globalScanStatus.getStatus(id) is None:
error.append("Scan '" + scaninfo[0] + "' is not actually running. A data consistency " + \
"error for this scan probably exists. <a href='/scandelete?id=" + \
id + "&confirm=1'>Click here to delete it.</a>")
errState = True
if not errState:
globalScanStatus.setStatus(id, "ABORT-REQUESTED")
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(pageid='SCANLIST',
stoppedscan=True,
errors=error,
docroot=self.docroot)
stopscanmulti.exposed = True
# Stop a scan.
def stopscan(self, id, cli=None):
global globalScanStatus
dbh = SpiderFootDb(self.config)
scaninfo = dbh.scanInstanceGet(id)
if scaninfo is None:
if not cli:
return self.error("Invalid scan ID.")
else:
return json.dumps(["ERROR", "Invalid scan ID."])
if globalScanStatus.getStatus(id) is None:
if not cli:
return self.error("That scan is not actually running. A data consistency " + \
"error for this scan probably exists. <a href='/scandelete?id=" + \
id + "&confirm=1'>Click here to delete it.</a>")
else:
return json.dumps(
["ERROR", "Scan doesn't appear to be running."])
if globalScanStatus.getStatus(id) == "ABORTED":
if not cli:
return self.error("The scan is already aborted.")
else:
return json.dumps(["ERROR", "Scan already aborted."])
if not globalScanStatus.getStatus(id) == "RUNNING":
if not cli:
return self.error("The running scan is currently in the state '" + \
globalScanStatus.getStatus(id) + "', please try again later or restart " + \
" SpiderFoot.")
else:
return json.dumps(
["ERROR", "Scan in an invalid state for stopping."])
globalScanStatus.setStatus(id, "ABORT-REQUESTED")
if not cli:
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(pageid='SCANLIST',
stoppedscan=True,
docroot=self.docroot,
errors=list())
else:
return json.dumps(["SUCCESS", ""])
stopscan.exposed = True
#
# DATA PROVIDERS
#
# Scan log data
def scanlog(self, id, limit=None, rowId=None, reverse=None):
dbh = SpiderFootDb(self.config)
data = dbh.scanLogs(id, limit, rowId, reverse)
retdata = []
for row in data:
generated = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0] / 1000))
retdata.append([
generated, row[1], row[2],
cgi.escape(unicode(row[3], errors='replace')), row[4]
])
return json.dumps(retdata)
scanlog.exposed = True
# Scan error data
def scanerrors(self, id, limit=None):
dbh = SpiderFootDb(self.config)
data = dbh.scanErrors(id, limit)
retdata = []
for row in data:
generated = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0] / 1000))
retdata.append([
generated, row[1],
cgi.escape(unicode(row[2], errors='replace'))
])
return json.dumps(retdata)
scanerrors.exposed = True
# Produce a list of scans
def scanlist(self):
dbh = SpiderFootDb(self.config)
data = dbh.scanInstanceList()
retdata = []
for row in data:
created = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[3]))
if row[4] != 0:
started = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[4]))
else:
started = "Not yet"
if row[5] != 0:
finished = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[5]))
else:
finished = "Not yet"
retdata.append([
row[0], row[1], row[2], created, started, finished, row[6],
row[7]
])
return json.dumps(retdata)
scanlist.exposed = True
# Basic information about a scan
def scanstatus(self, id):
dbh = SpiderFootDb(self.config)
data = dbh.scanInstanceGet(id)
created = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[2]))
started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[3]))
ended = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[4]))
retdata = [data[0], data[1], created, started, ended, data[5]]
return json.dumps(retdata)
scanstatus.exposed = True
# Summary of scan results
def scansummary(self, id, by):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultSummary(id, by)
retdata = []
for row in data:
if row[0] == "ROOT":
continue
lastseen = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[2]))
retdata.append([row[0], row[1], lastseen, row[3], row[4]])
return json.dumps(retdata)
scansummary.exposed = True
# Event results for a scan
def scaneventresults(self, id, eventType, filterfp=False):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultEvent(id, eventType, filterfp)
retdata = []
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0]))
escapeddata = cgi.escape(row[1])
escapedsrc = cgi.escape(row[2])
retdata.append([
lastseen, escapeddata, escapedsrc, row[3], row[5], row[6],
row[7], row[8], row[13], row[14], row[4]
])
return json.dumps(retdata, ensure_ascii=False)
scaneventresults.exposed = True
# Unique event results for a scan
def scaneventresultsunique(self, id, eventType, filterfp=False):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultEventUnique(id, eventType, filterfp)
retdata = []
for row in data:
escaped = cgi.escape(row[0])
retdata.append([escaped, row[1], row[2]])
return json.dumps(retdata, ensure_ascii=False)
scaneventresultsunique.exposed = True
# Search
def search(self, id=None, eventType=None, value=None):
retdata = self.searchBase(id, eventType, value)
return json.dumps(retdata, ensure_ascii=False)
search.exposed = True
# Historical data for the scan, graphs will be rendered in JS
def scanhistory(self, id):
dbh = SpiderFootDb(self.config)
data = dbh.scanResultHistory(id)
return json.dumps(data, ensure_ascii=False)
scanhistory.exposed = True
def scanelementtypediscovery(self, id, eventType):
sf = SpiderFoot(self.config)
dbh = SpiderFootDb(self.config)
pc = dict()
datamap = dict()
# Get the events we will be tracing back from
leafSet = dbh.scanResultEvent(id, eventType)
[datamap, pc] = dbh.scanElementSourcesAll(id, leafSet)
# Delete the ROOT key as it adds no value from a viz perspective
del pc['ROOT']
retdata = dict()
retdata['tree'] = sf.dataParentChildToTree(pc)
retdata['data'] = datamap
return json.dumps(retdata, ensure_ascii=False)
scanelementtypediscovery.exposed = True
message = traceback.format_exception(*sys.exc_info())
print(message)
logger.exception("%s" % err)
resp = ServiceError("%s" % err)
return resp(environ, start_response)
LOGGER.debug("unknown side: %s" % path)
resp = NotFound("Couldn't find the side you asked for!")
return resp(environ, start_response)
if __name__ == '__main__':
root = './'
lookup = TemplateLookup(directories=[root + 'Templates', root + 'htdocs'],
module_directory=root + 'modules',
input_encoding='utf-8',
output_encoding='utf-8')
usernamePasswords = {"user1": "1", "user2": "2"}
passwordEndPointIndex = 0 # what is this, and what does its value mean?
# JWKS: JSON Web Key
jwksFileName = "static/jwks.json"
# parse the parameters
parser = argparse.ArgumentParser()
parser.add_argument('-c', dest='config')
parser.add_argument('-d', dest='debug', action='store_true')
args = parser.parse_args()
import importlib
import os
import sys
import codecs
from mako.lookup import TemplateLookup
MODULE_PATH = os.getenv(
'MODULE_PATH',
os.path.realpath(os.path.join(os.path.dirname(__file__), '../modules')))
TEMPLATE_PATH = os.getenv(
'TEMPLATE_PATH',
os.path.realpath(
os.path.join(os.path.dirname(__file__), 'boilerplate_data')))
VERSION = '1.2'
TEMPLATES = TemplateLookup(directories=[TEMPLATE_PATH])
def u8(s):
return s.decode('utf-8')
def gitconfig(entry):
return u8(
subprocess.check_output('git config -z --get %s' % entry,
shell=True)[:-1])
def write(target, contents):
if not os.path.isdir(os.path.dirname(target)):
os.makedirs(os.path.dirname(target))
class AggregatePages(object):
@cherrypy.expose
def index(self):
return self.callstacks()
templates_dir = os.path.join(os.getcwd(),'static','templates')
template_lookup = TemplateLookup(directories=[templates_dir,])
@cherrypy.expose
def callstacks(self, id=None, **kwargs):
table_kwargs, filter_kwargs = parse_kwargs(kwargs)
if 'id' in filter_kwargs:
filter_kwargs.pop('id')
for k in filter_kwargs:
filter_kwargs[k] = str(filter_kwargs[k])
if id:
call_stack, total, filtered = json_aggregate_item(db.CallStack, filter_kwargs, id)
if call_stack == None:
raise cherrypy.HTTPError(404)
call_stack[1] = str(call_stack[1]) #unicode throws off template when casting dict as js obj
call_stack[2] = int(call_stack[2]) #convert long to int
mytemplate = Template(filename=os.path.join(self.templates_dir,'aggregatecallstack.html'), lookup=self.template_lookup)
return mytemplate.render(call_stack=call_stack, kwargs=filter_kwargs)
else:
mytemplate = Template(filename=os.path.join(self.templates_dir,'aggregatecallstacks.html'), lookup=self.template_lookup)
return mytemplate.render(kwargs=filter_kwargs)
@cherrypy.expose
def sqlstatements(self, id=None, **kwargs):
table_kwargs, filter_kwargs = parse_kwargs(kwargs)
if 'id' in filter_kwargs:
filter_kwargs.pop('id')
for k in filter_kwargs:
filter_kwargs[k] = str(filter_kwargs[k])
if id:
statement, total, filtered = json_aggregate_item(db.SQLStatement, filter_kwargs, id)
if statement == None:
raise cherrypy.HTTPError(404)
statement[1] = str(statement[1]) #unicode throws off template when casting dict as js obj
statement[2] = int(statement[2]) #convert long to int
mytemplate = Template(filename=os.path.join(self.templates_dir,'aggregatesql.html'), lookup=self.template_lookup)
return mytemplate.render(statement=statement, kwargs=filter_kwargs)
else:
mytemplate = Template(filename=os.path.join(self.templates_dir,'aggregatesqls.html'), lookup=self.template_lookup)
return mytemplate.render(kwargs=filter_kwargs)
@cherrypy.expose
def fileaccesses(self, id=None, **kwargs):
table_kwargs, filter_kwargs = parse_kwargs(kwargs)
if 'id' in filter_kwargs:
filter_kwargs.pop('id')
for k in filter_kwargs:
filter_kwargs[k] = str(filter_kwargs[k])
if id:
file_access, total, filtered = json_aggregate_item(db.FileAccess, filter_kwargs, id)
if file_access == None:
raise cherrypy.HTTPError(404)
file_access[1] = str(file_access[1]) #unicode throws off template when casting dict as js obj
file_access[2] = int(file_access[2]) #convert long to int
mytemplate = Template(filename=os.path.join(self.templates_dir,'aggregatefileaccess.html'), lookup=self.template_lookup)
return mytemplate.render(file_access=file_access, kwargs=filter_kwargs)
else:
mytemplate = Template(filename=os.path.join(self.templates_dir,'aggregatefileaccesses.html'), lookup=self.template_lookup)
return mytemplate.render(kwargs=filter_kwargs)
.. moduleauthor:: `Marie FETIVEAU <github.com/mfe>`_
"""
__version__ = "0.0"
import cherrypy
from mako.lookup import TemplateLookup
import os
import ntpath
import traceback
from plotThatLut import plot_that_lut
from utils import matplotlib_helper as mplh
mplh.WEB_MODE = True
CURRENT_DIR = os.path.dirname(os.path.abspath(__file__))
MY_LOOKUP = TemplateLookup(directories=[os.path.join(CURRENT_DIR, 'html')])
class Application(object):
"""CherryPy Application
"""
@cherrypy.expose
def index(self):
"""ColorPipe-tools index page
"""
mytemplate = MY_LOOKUP.get_template("index.html")
return mytemplate.render()
@cherrypy.expose
from otest.rp.setup import as_arg_setup
from otest.rp.tool import WebTester
try:
from requests.packages import urllib3
except ImportError:
pass
else:
urllib3.disable_warnings()
__author__ = 'roland'
ROOT = './'
LOOKUP = TemplateLookup(directories=[ROOT + 'htdocs'],
module_directory=ROOT + 'modules',
input_encoding='utf-8',
output_encoding='utf-8')
logger = logging.getLogger("")
def setup_logging(logfile_name, log_level=logging.DEBUG):
hdlr = logging.FileHandler(logfile_name)
base_formatter = logging.Formatter(
"%(asctime)s %(name)s:%(levelname)s %(message)s")
hdlr.setFormatter(base_formatter)
logger.addHandler(hdlr)
logger.setLevel(log_level)
import re
import httplib
import json
import cherrypy
import gdg
import bcrypt
from urlparse import urljoin, urlparse
from mako.template import Template
from mako.lookup import TemplateLookup
from gdg.data import *
# Content is relative to the base directory, not the module directory.
current_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
virtual_dir = ''
templates = TemplateLookup(directories=['html'], strict_undefined=True)
application = None
class ImageModel(object):
def __init__(self, **entries):
self.__dict__.update(entries)
def get_relative_path(base, path):
if path == None:
return None
if not path == "":
path = os.path.relpath(path, current_dir)
return urljoin(base, path.replace('\\', '/'))
def __init__(self, template_directory):
self.lookup = TemplateLookup(directories=[template_directory],
strict_undefined=True)
def resource_path(relative):
"""Makes relative to package"""
return pkg_resources.resource_filename(__name__, relative)
#TODO: fix support here for template lookups, internal, user provided
#template_cache_dir = config.template_cache_dir
template_cache_dir = "cache"
#disable cache for cleaner folder structure
#TODO: Also try for Cisco build here
lookup = TemplateLookup(directories=[resource_path("")],
#module_directory= template_cache_dir,
cache_type='memory',
cache_enabled=True,
)
try:
import autonetkit_cisco # test if can import, if not present will fail and not add to template path
cisco_templates = pkg_resources.resource_filename("autonetkit_cisco", "")
lookup.directories.append(cisco_templates)
except ImportError:
pass # Cisco ANK not present
#TODO: make a render class, that caches traversed folders for speed
#TODO: Add support for both src template and src folder (eg for quagga, servers)
def render_node(node, folder_cache):
log.debug("Rendering %s" % node)
def load_environment(global_conf, app_conf):
"""
Configure the Pylons environment via the ``pylons.config``
object
"""
# Pylons paths
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
paths = dict(root=root,
controllers=os.path.join(root, 'controllers'),
static_files=os.path.join(root, 'public'),
templates=[
app_conf.get('custom_templates',
os.path.join(root, 'templates')),
os.path.join(root, 'templates')
])
# Initialize config with the basic options
config.init_app(global_conf, app_conf, package='linotp', paths=paths)
config['linotp.root'] = root
config['routes.map'] = make_map()
config['pylons.app_globals'] = app_globals.Globals()
config['pylons.h'] = linotp.lib.helpers
## add per token a location for the mako template lookup
## @note: the location is defined in the .ini file by
## the entry [linotpTokenModules]
directories = paths['templates']
## add a template path for every token
modules = get_token_module_list()
for module in modules:
mpath = os.path.dirname(module.__file__)
directories.append(mpath)
## add a template path for every resolver
modules = get_resolver_module_list()
for module in modules:
mpath = os.path.dirname(module.__file__)
directories.append(mpath)
unique_directories = _uniqify_list(directories)
log.debug("[load_environment] Template directories: %r" %
unique_directories)
config['pylons.app_globals'].mako_lookup = TemplateLookup(
directories=unique_directories,
error_handler=handle_mako_error,
module_directory=os.path.join(app_conf['cache_dir'], 'templates'),
input_encoding='utf-8',
default_filters=['escape'],
imports=['from webhelpers.html import escape'])
# Setup the SQLAlchemy database engine
# If we load the linotp.model here, the pylons.config is loaded with
# the entries from the config file. if it is loaded at the top of the file,
#the pylons.config does not contain the config file, yet.
from linotp.model import init_model
engine = engine_from_config(config, 'sqlalchemy.')
init_model(engine)
# CONFIGURATION OPTIONS HERE (note: all config options will override
# any Pylons config options)
from linotp.lib.audit.base import getAudit
audit = getAudit()
config['audit'] = audit
## setup Security provider definition
try:
log.debug('[load_environment] loading token list definition')
g = config['pylons.app_globals']
g.security_provider.load_config(config)
except Exception as e:
log.error("Failed to load security provider definition: %r" % e)
raise e
## load the list of tokenclasses
try:
log.debug('[load_environment] loading token list definition')
(tcl, tpl) = get_token_class_list()
config['tokenclasses'] = tcl
g.setTokenclasses(tcl)
config['tokenprefixes'] = tpl
g.setTokenprefixes(tpl)
except Exception as e:
log.error("Failed to load token class list: %r" % e)
raise e
## load the list of resolvers
try:
log.debug('[load_environment] loading resolver list definition')
(rclass, rname) = get_resolver_class_list()
## make this globaly avaliable
g.setResolverClasses(rclass)
g.setResolverTypes(rname)
except Exception as exx:
log.error("Faild to load the list of resolvers: %r" % exx)
raise exx
## get the help url
url = config.get("linotpHelp.url", None)
if url is None:
version = pkg_resources.get_distribution("linotp").version
# First try to get the help for this specific version
url = "http://linotp.org/doc/%s/index.html" % version
config['help_url'] = url
log.debug("[load_environment] done")
return
def _write_httpd_conf(pubs,
default_pubs,
runtime_dir,
log_dir,
template_dir,
cache_dir,
cache_size,
host,
port,
sroot,
fragment=False,
allow_refresh=False,
ssl_cert_file="",
ssl_key_file="",
ssl_cert_chain_file=""):
"""Writes the webserver configuration for the depot.
pubs repository and publisher information, a list in the form
[(publisher_prefix, repo_dir, repo_prefix,
writable_root), ... ]
default_pubs default publishers, per repository, a list in the form
[(default_publisher_prefix, repo_dir, repo_prefix) ... ]
runtime_dir where we write httpd.conf files
log_dir where Apache should write its log files
template_dir where we find our Mako templates
cache_dir where Apache should write its cache and wsgi search idx
cache_size how large our cache can grow
host our hostname, needed to set ServerName properly
port the port on which Apache should listen
sroot the prefix into the server namespace,
ignored if fragment==False
fragment True if we should only write a file to drop into conf.d/
(i.e. a partial server configuration)
allow_refresh True if we allow the 'refresh' or 'refresh-indexes'
admin/0 operations
The URI namespace we create on the web server looks like this:
<sroot>/<repo_prefix>/<publisher>/<file, catalog etc.>/<version>/
<sroot>/<repo_prefix>/<file, catalog etc.>/<version>/
'sroot' is only used when the Apache server is serving other content
and we want to separate pkg(5) resources from the other resources
provided.
'repo_prefix' exists so that we can disambiguate between multiple
repositories that provide the same publisher.
'ssl_cert_file' the location of the server certificate file.
'ssl_key_file' the location of the server key file.
'ssl_cert_chain_file' the location of the certificate chain file if the
the server certificate is not signed by the top level CA.
"""
try:
# check our hostname
socket.getaddrinfo(host, None)
# Apache needs IPv6 addresses wrapped in square brackets
if ":" in host:
host = "[{0}]".format(host)
# check our directories
dirs = [runtime_dir]
if not fragment:
dirs.append(log_dir)
if cache_dir:
dirs.append(cache_dir)
for dir in dirs + [template_dir]:
if os.path.exists(dir) and not os.path.isdir(dir):
raise DepotException(_("{0} is not a directory").format(dir))
for dir in dirs:
misc.makedirs(dir)
# check our port
if not fragment:
try:
num = int(port)
if num <= 0 or num >= 65535:
raise DepotException(_("invalid port: {0}").format(port))
except ValueError:
raise DepotException(_("invalid port: {0}").format(port))
# check our cache size
try:
num = int(cache_size)
if num < 0:
raise DepotException(
_("invalid cache size: "
"{0}").format(num))
except ValueError:
raise DepotException(
_("invalid cache size: {0}").format(cache_size))
httpd_conf_template_path = os.path.join(template_dir,
DEPOT_HTTP_TEMPLATE)
fragment_conf_template_path = os.path.join(template_dir,
DEPOT_FRAGMENT_TEMPLATE)
# we're disabling unicode here because we want Mako to
# passthrough any filesystem path names, whatever the
# original encoding.
conf_lookup = TemplateLookup(directories=[template_dir])
disable_unicode = True if six.PY2 else False
if fragment:
conf_template = Template(filename=fragment_conf_template_path,
disable_unicode=disable_unicode,
lookup=conf_lookup)
conf_path = os.path.join(runtime_dir, DEPOT_FRAGMENT_FILENAME)
else:
conf_template = Template(filename=httpd_conf_template_path,
disable_unicode=disable_unicode,
lookup=conf_lookup)
conf_path = os.path.join(runtime_dir, DEPOT_HTTP_FILENAME)
conf_text = conf_template.render(
pubs=pubs,
default_pubs=default_pubs,
log_dir=log_dir,
cache_dir=cache_dir,
cache_size=cache_size,
runtime_dir=runtime_dir,
template_dir=template_dir,
ipv6_addr="::1",
host=host,
port=port,
sroot=sroot,
allow_refresh=allow_refresh,
ssl_cert_file=ssl_cert_file,
ssl_key_file=ssl_key_file,
ssl_cert_chain_file=ssl_cert_chain_file)
with open(conf_path, "w") as conf_file:
conf_file.write(conf_text)
except (socket.gaierror, UnicodeError) as err:
# socket.getaddrinfo raise UnicodeDecodeError in Python 3
# for some input, such as '.'
raise DepotException(
_("Unable to write Apache configuration: {host}: "
"{err}").format(**locals()))
except (OSError, IOError, EnvironmentError, apx.ApiException) as err:
traceback.print_exc()
raise DepotException(
_("Unable to write depot_httpd.conf: {0}").format(err))
def get_lookup():
mylookup = TemplateLookup(directories=['./templates'],
output_encoding='utf-8',
encoding_errors='replace')
mytemplate = mylookup.get_template("base.html")
return None
import json
from django.conf import settings
from django.http import HttpResponse
from django.template.context import Context
from mako.lookup import TemplateLookup
from common.log import logger
"""
mylookup 也可以单独使用：
@Example:
mako_temp = mylookup.get_template(template_name)
mako_temp.render(**data)
"""
mylookup = TemplateLookup(
directories=settings.MAKO_TEMPLATE_DIR,
module_directory=settings.MAKO_TEMPLATE_MODULE_DIR,
output_encoding='utf-8',
input_encoding='utf-8',
encoding_errors='replace',
collection_size=500,
)
def render_mako(template_name, dictionary={}, context_instance=None):
"""
render the mako template and return the HttpResponse
@param template_name: 模板名字
@param dictionary: context字典
@param context_instance: 初始化context，如果要使用 TEMPLATE_CONTEXT_PROCESSORS，则要使用RequestContext(request)
@note: 因为返回是HttpResponse，所以这个方法也适合给ajax用(dataType不是json的ajax)
@Example:
render_mako('mako_temp.html',{'form':form})
# add current folder to system path
sys.path.insert(0, os.getcwd())
imports = []
try:
# you can use mysite module to hold global variables
import mysite
imports.append('import mysite')
except ImportError:
print('Please create module mysite.py to keep global vars!')
lookup = TemplateLookup(directories=['.'],
input_encoding='utf-8',
output_encoding='utf-8',
encoding_errors='replace',
imports=imports)
class StoppableHTTPServer(HTTPServer):
"""HTTPServer that can be interrupted by pressing Ctrl-C"""
def run(self):
(hostaddr, port) = self.socket.getsockname()
print("Serving HTTP on {} port {} ...".format(hostaddr, port))
h = '127.0.0.1' if hostaddr == '0.0.0.0' else hostaddr
print("Open http://{}:{} in your browser!".format(h, port))
try:
self.serve_forever()
except KeyboardInterrupt:
# Clean-up server (close socket, etc.)
from osf.models.base import BaseModel, ObjectIDMixin
from osf.models.external import ExternalAccount
from osf.models.node import AbstractNode
from osf.models.user import OSFUser
from osf.utils.datetime_aware_jsonfield import DateTimeAwareJSONField
from website import settings
from addons.base import logger, serializer
from website.oauth.signals import oauth_complete
lookup = TemplateLookup(
directories=[settings.TEMPLATES_PATH],
default_filters=[
'unicode', # default filter; must set explicitly when overriding
# FIXME: Temporary workaround for data stored in wrong format in DB. Unescape it before it
# gets re-escaped by Markupsafe. See [#OSF-4432]
'temp_ampersand_fixer',
'h',
],
imports=[
# FIXME: Temporary workaround for data stored in wrong format in DB. Unescape it before it
# gets re-escaped by Markupsafe. See [#OSF-4432]
'from website.util.sanitize import temp_ampersand_fixer',
])
class BaseAddonSettings(ObjectIDMixin, BaseModel):
deleted = models.BooleanField(default=False)
class Meta:
abstract = True
@property
def build_templatelookup(args):
# Set up the location the templates will be served out of.
return TemplateLookup(directories=[args.filedir],
module_directory=args.cachedir,
collection_size=100)
#Kaithem Automation is free software: you can redistribute it and/or modify
#it under the terms of the GNU General Public License as published by
#the Free Software Foundation, version 3.
#Kaithem Automation is distributed in the hope that it will be useful,
#but WITHOUT ANY WARRANTY; without even the implied warranty of
#MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
#GNU General Public License for more details.
#You should have received a copy of the GNU General Public License
#along with Kaithem Automation. If not, see <http://www.gnu.org/licenses/>.
from mako.template import Template
from mako.lookup import TemplateLookup
import auth
import cherrypy
_Lookup = TemplateLookup(directories=['pages'])
get_template = _Lookup.get_template
#Redirect user to an error message if he does not have the proper permission.
def require(permission):
if (not 'auth' in cherrypy.request.cookie
) or cherrypy.request.cookie['auth'].value not in auth.Tokens:
raise cherrypy.HTTPRedirect("/login/")
if not auth.checkTokenPermission(cherrypy.request.cookie['auth'].value,
permission):
raise cherrypy.HTTPRedirect("/errors/permissionerror")
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--out',
help='Path to the output directory',
required=True)
parser.add_argument('--staging',
help='Path to the staging directory',
required=True)
parser.add_argument('--zircon-user-build',
help='Path to the Zircon "user" build directory',
required=True)
parser.add_argument('--zircon-tool-build',
help='Path to the Zircon "tools" build directory',
required=True)
parser.add_argument('--debug',
help='Whether to print out debug information',
action='store_true')
args = parser.parse_args()
out_dir = os.path.abspath(args.out)
shutil.rmtree(os.path.join(out_dir, 'config'), True)
shutil.rmtree(os.path.join(out_dir, 'fidl'), True)
shutil.rmtree(os.path.join(out_dir, 'lib'), True)
shutil.rmtree(os.path.join(out_dir, 'sysroot'), True)
shutil.rmtree(os.path.join(out_dir, 'tool'), True)
debug = args.debug
# Generate package descriptions through Zircon's build.
zircon_dir = os.path.abspath(args.staging)
shutil.rmtree(zircon_dir, True)
if debug:
print('Building Zircon in: %s' % zircon_dir)
make_args = [
'make',
'packages',
'BUILDDIR=%s' % zircon_dir,
]
env = {}
env['PATH'] = os.environ['PATH']
if not debug:
env['QUIET'] = '1'
subprocess.check_call(make_args, cwd=ZIRCON_ROOT, env=env)
# Parse package definitions.
packages = []
with open(os.path.join(zircon_dir, 'export', 'manifest'), 'r') as manifest:
package_files = map(lambda line: line.strip(), manifest.readlines())
for file in package_files:
with open(os.path.join(zircon_dir, 'export', file), 'r') as pkg_file:
packages.append(parse_package(pkg_file.readlines()))
if debug:
print('Found %s packages:' % len(packages))
names = sorted(map(lambda p: p['package']['name'], packages))
for name in names:
print(' - %s' % name)
# Generate some GN glue for each package.
context = GenerationContext(
out_dir,
ZIRCON_ROOT,
os.path.abspath(args.zircon_user_build),
os.path.abspath(args.zircon_tool_build),
TemplateLookup(directories=[SCRIPT_DIR]),
)
for package in packages:
name = package['package']['name']
type = package['package']['type']
arch = package['package']['arch']
if name == 'c':
generate_sysroot(package, context)
print('Generated sysroot')
continue
if name in SYSROOT_PACKAGES:
print('Ignoring sysroot part: %s' % name)
continue
if type == 'tool':
generate_host_tool(package, context)
elif type == 'lib':
if arch == 'src':
type = 'source'
generate_source_library(package, context)
else:
type = 'prebuilt'
generate_compiled_library(package, context)
elif type == 'fidl':
generate_fidl_library(package, context)
else:
print('(%s) Unsupported package type: %s/%s, skipping' %
(name, type, arch))
continue
if debug:
print('Processed %s (%s)' % (name, type))
board_path = os.path.join(zircon_dir, 'export', 'all-boards.list')
with open(board_path, 'r') as board_file:
package = parse_package(board_file.readlines())
generate_board_list(package, context)
def create_lookup(self):
"""Create a template lookup."""
self.lookup = TemplateLookup(
directories=self.directories,
module_directory=self.cache_dir,
output_encoding='utf-8')
def __init__(self):
self.lookup = TemplateLookup(directories=['.'],
output_encoding='utf-8',
encoding_errors='replace')
def __init__(self, *a, **kwargs):
from mako.lookup import TemplateLookup
self._lookup = TemplateLookup(*a, **kwargs)
class SpiderFootWebUi:
lookup = TemplateLookup(directories=[''])
defaultConfig = dict()
config = dict()
token = None
docroot = ''
def __init__(self, config):
"""Initialize web server
Args:
config: TBD
"""
if not isinstance(config, dict):
raise TypeError("config is %s; expected dict()" % type(config))
if not config:
raise ValueError("config is empty")
self.defaultConfig = deepcopy(config)
dbh = SpiderFootDb(self.defaultConfig)
# 'config' supplied will be the defaults, let's supplement them
# now with any configuration which may have previously been
# saved.
sf = SpiderFoot(self.defaultConfig)
self.config = sf.configUnserialize(dbh.configGet(), self.defaultConfig)
if self.config['__webaddr'] == "0.0.0.0": # nosec
addr = "<IP of this host>"
else:
addr = self.config['__webaddr']
self.docroot = self.config['__docroot'].rstrip('/')
cherrypy.config.update({
'error_page.404': self.error_page_404,
'request.error_response': self.error_page
})
secure_headers = SecureHeaders(
server="server",
cache=False,
csp=
"default-src 'self' ; script-src 'self' 'unsafe-inline' blob: ; style-src 'self' 'unsafe-inline' ; img-src 'self' data:"
)
cherrypy.config.update({
"tools.response_headers.on":
True,
"tools.response_headers.headers":
secure_headers.cherrypy()
})
if (cherrypy.server.ssl_certificate is None
or cherrypy.server.ssl_private_key is None):
url = "http://%s:%s%s" % (addr, self.config['__webport'],
self.docroot)
else:
url = "https://%s:%s%s" % (addr, self.config['__webport'],
self.docroot)
print("")
print("")
print("*************************************************************")
print(" Use SpiderFoot by starting your web browser of choice and ")
print(" browse to %s" % url)
print("*************************************************************")
print("")
print("")
def error_page(self):
"""Error page"""
cherrypy.response.status = 500
if self.config['_debug']:
cherrypy.response.body = _cperror.get_error_page(
status=500, traceback=_cperror.format_exc())
else:
cherrypy.response.body = '<html><body>Error</body></html>'
def error_page_404(self, status, message, traceback, version):
"""Error page 404
Args:
status: TBD
message: TBD
traceback: TBD
version: TBD
"""
templ = Template(filename='dyn/error.tmpl', lookup=self.lookup)
return templ.render(message='Not Found',
docroot=self.docroot,
status=status)
def cleanUserInput(self, inputList):
"""Sanitize user input, poorly.
Args:
inputList (list): TBD
Returns:
list: sanitized input
Raises:
TypeError: inputList type was invalid
"""
if not isinstance(inputList, list):
raise TypeError("inputList is %s; expected list()" %
type(inputList))
ret = list()
for item in inputList:
c = html.escape(item, True)
c = c.replace('\'', '"')
# We don't actually want & translated to &
c = c.replace("&", "&").replace(""", "\"")
ret.append(c)
return ret
def searchBase(self, id=None, eventType=None, value=None):
"""Search
Args:
id: TBD
eventType: TBD
value: TBD
Returns:
list: search results
"""
retdata = []
regex = ""
if [id, eventType, value].count('') == 3 or [id, eventType, value
].count(None) == 3:
return retdata
if value.startswith("/") and value.endswith("/"):
regex = value[1:len(value) - 1]
value = ""
value = value.replace('*', '%')
if value in [None, ""] and regex in [None, ""]:
value = "%"
regex = ""
dbh = SpiderFootDb(self.config)
criteria = {
'scan_id': None if id == '' else id,
'type': None if eventType == '' else eventType,
'value': None if value == '' else value,
'regex': None if regex == '' else regex
}
try:
data = dbh.search(criteria)
except Exception:
return retdata
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0]))
escapeddata = html.escape(row[1])
escapedsrc = html.escape(row[2])
retdata.append([
lastseen, escapeddata, escapedsrc, row[3], row[5], row[6],
row[7], row[8], row[10], row[11], row[4], row[13], row[14]
])
return retdata
#
# USER INTERFACE PAGES
#
def scaneventresultexport(self, id, type, dialect="excel"):
"""Get scan event result data in CSV format
Args:
id (str): scan ID
type (str): TBD
dialect (str): TBD
Returns:
string: results in CSV format
"""
dbh = SpiderFootDb(self.config)
data = dbh.scanResultEvent(id, type)
fileobj = StringIO()
parser = csv.writer(fileobj, dialect=dialect)
parser.writerow(["Updated", "Type", "Module", "Source", "F/P", "Data"])
for row in data:
if row[4] == "ROOT":
continue
lastseen = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0]))
datafield = str(row[1]).replace("<SFURL>",
"").replace("</SFURL>", "")
parser.writerow([
lastseen,
str(row[4]),
str(row[3]),
str(row[2]), row[13], datafield
])
cherrypy.response.headers[
'Content-Disposition'] = "attachment; filename=SpiderFoot.csv"
cherrypy.response.headers['Content-Type'] = "application/csv"
cherrypy.response.headers['Pragma'] = "no-cache"
return fileobj.getvalue().encode("utf-8")
scaneventresultexport.exposed = True
def scaneventresultexportmulti(self, ids, dialect="excel"):
"""Get scan event result data in CSV format for multiple scans
Args:
ids (str): comma separated list of scan IDs
dialect (str): TBD
Returns:
string: results in CSV format
"""
dbh = SpiderFootDb(self.config)
scaninfo = dict()
data = list()
for id in ids.split(','):
scaninfo[id] = dbh.scanInstanceGet(id)
data = data + dbh.scanResultEvent(id)
fileobj = StringIO()
parser = csv.writer(fileobj, dialect=dialect)
parser.writerow([
"Scan Name", "Updated", "Type", "Module", "Source", "F/P", "Data"
])
for row in data:
if row[4] == "ROOT":
continue
lastseen = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0]))
datafield = str(row[1]).replace("<SFURL>",
"").replace("</SFURL>", "")
parser.writerow([
scaninfo[row[12]][0], lastseen,
str(row[4]),
str(row[3]),
str(row[2]), row[13], datafield
])
cherrypy.response.headers[
'Content-Disposition'] = "attachment; filename=SpiderFoot.csv"
cherrypy.response.headers['Content-Type'] = "application/csv"
cherrypy.response.headers['Pragma'] = "no-cache"
return fileobj.getvalue().encode("Utf-8")
scaneventresultexportmulti.exposed = True
# Get search result data in CSV format
def scansearchresultexport(self,
id,
eventType=None,
value=None,
dialect="excel"):
data = self.searchBase(id, eventType, value)
fileobj = StringIO()
parser = csv.writer(fileobj, dialect=dialect)
parser.writerow(["Updated", "Type", "Module", "Source", "F/P", "Data"])
if not data:
return None
for row in data:
if row[10] == "ROOT":
continue
datafield = str(row[1]).replace("<SFURL>",
"").replace("</SFURL>", "")
parser.writerow([
row[0],
str(row[10]),
str(row[3]),
str(row[2]), row[11], datafield
])
cherrypy.response.headers[
'Content-Disposition'] = "attachment; filename=SpiderFoot.csv"
cherrypy.response.headers['Content-Type'] = "application/csv"
cherrypy.response.headers['Pragma'] = "no-cache"
return fileobj.getvalue().encode("Utf-8")
scansearchresultexport.exposed = True
# Export results from multiple scans in JSON format
def scanexportjsonmulti(self, ids):
dbh = SpiderFootDb(self.config)
scaninfo = list()
scan_name = ""
for id in ids.split(','):
scan = dbh.scanInstanceGet(id)
if scan is None:
continue
scan_name = scan[0]
for row in dbh.scanResultEvent(id):
lastseen = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0]))
event_data = str(row[1]).replace("<SFURL>",
"").replace("</SFURL>", "")
source_data = str(row[2])
source_module = str(row[3])
event_type = row[4]
false_positive = row[13]
if event_type == "ROOT":
continue
scaninfo.append({
"data": event_data,
"event_type": event_type,
"module": source_module,
"source_data": source_data,
"false_positive": false_positive,
"last_seen": lastseen,
"scan_name": scan_name,
"scan_target": scan[1]
})
if len(ids.split(',')) > 1 or scan_name == "":
fname = "SpiderFoot.json"
else:
fname = scan_name + "-SpiderFoot.json"
cherrypy.response.headers[
'Content-Disposition'] = "attachment; filename=" + fname
cherrypy.response.headers[
'Content-Type'] = "application/json; charset=utf-8"
cherrypy.response.headers['Pragma'] = "no-cache"
return json.dumps(scaninfo).encode("utf-8")
scanexportjsonmulti.exposed = True
# Export entities from scan results for visualising
def scanviz(self, id, gexf="0"):
dbh = SpiderFootDb(self.config)
sf = SpiderFoot(self.config)
data = dbh.scanResultEvent(id, filterFp=True)
scan = dbh.scanInstanceGet(id)
root = scan[1]
if gexf != "0":
cherrypy.response.headers[
'Content-Disposition'] = "attachment; filename=SpiderFoot.gexf"
cherrypy.response.headers['Content-Type'] = "application/gexf"
cherrypy.response.headers['Pragma'] = "no-cache"
return sf.buildGraphGexf([root], "SpiderFoot Export", data)
else:
return sf.buildGraphJson([root], data)
scanviz.exposed = True
# Export entities results from multiple scans in GEXF format
def scanvizmulti(self, ids, gexf="1"):
dbh = SpiderFootDb(self.config)
sf = SpiderFoot(self.config)
data = list()
roots = list()
for id in ids.split(','):
data = data + dbh.scanResultEvent(id, filterFp=True)
roots.append(dbh.scanInstanceGet(id)[1])
if gexf != "0":
cherrypy.response.headers[
'Content-Disposition'] = "attachment; filename=SpiderFoot.gexf"
cherrypy.response.headers['Content-Type'] = "application/gexf"
cherrypy.response.headers['Pragma'] = "no-cache"
return sf.buildGraphGexf(roots, "SpiderFoot Export", data)
else:
# Not implemented yet
return None
scanvizmulti.exposed = True
def scanopts(self, id):
"""Configuration used for a scan
Args:
id: scan ID
"""
ret = dict()
dbh = SpiderFootDb(self.config)
ret['config'] = dbh.scanConfigGet(id)
ret['configdesc'] = dict()
for key in list(ret['config'].keys()):
if ':' not in key:
ret['configdesc'][key] = self.config['__globaloptdescs__'][key]
else:
[modName, modOpt] = key.split(':')
if modName not in list(self.config['__modules__'].keys()):
continue
if modOpt not in list(self.config['__modules__'][modName]
['optdescs'].keys()):
continue
ret['configdesc'][key] = self.config['__modules__'][modName][
'optdescs'][modOpt]
meta = dbh.scanInstanceGet(id)
if not meta:
return json.dumps([])
if meta[3] != 0:
started = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(meta[3]))
else:
started = "Not yet"
if meta[4] != 0:
finished = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(meta[4]))
else:
finished = "Not yet"
ret['meta'] = [meta[0], meta[1], meta[2], started, finished, meta[5]]
return json.dumps(ret)
scanopts.exposed = True
def rerunscan(self, id):
"""Rerun a scan
Args:
id (str): scan ID
"""
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
info = dbh.scanInstanceGet(id)
if not info:
return self.error("Invalid scan ID.")
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if len(scanconfig) == 0:
return self.error("Something went wrong internally.")
modlist = scanconfig['_modulesenabled'].split(',')
if "sfp__stor_stdout" in modlist:
modlist.remove("sfp__stor_stdout")
targetType = sf.targetType(scantarget)
if targetType is None:
# It must then be a name, as a re-run scan should always have a clean
# target. Put quotes around the target value and try to determine the
# target type again.
targetType = sf.targetType(f'"{scantarget}"')
if targetType != "HUMAN_NAME":
scantarget = scantarget.lower()
# Start running a new scan
scanId = sf.genScanInstanceGUID()
try:
p = mp.Process(target=SpiderFootScanner,
args=(scanname, scanId, scantarget, targetType,
modlist, cfg))
p.daemon = True
p.start()
except BaseException as e:
print("[-] Scan [%s] failed: %s" % (scanId, e))
return self.error("Scan [%s] failed: %s" % (scanId, e))
# Wait until the scan has initialized
while dbh.scanInstanceGet(scanId) is None:
print("[info] Waiting for the scan to initialize...")
time.sleep(1)
raise cherrypy.HTTPRedirect(f"scaninfo?id={scanId}", status=302)
rerunscan.exposed = True
def rerunscanmulti(self, ids):
"""Rerun scans
Args:
ids (str): comma separated list of scan IDs
"""
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
for id in ids.split(","):
info = dbh.scanInstanceGet(id)
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if len(scanconfig) == 0:
return self.error("Something went wrong internally.")
modlist = scanconfig['_modulesenabled'].split(',')
if "sfp__stor_stdout" in modlist:
modlist.remove("sfp__stor_stdout")
targetType = sf.targetType(scantarget)
if targetType is None:
# Should never be triggered for a re-run scan..
return self.error(
"Invalid target type. Could not recognize it as a human name, IP address, IP subnet, ASN, domain name or host name."
)
# Start running a new scan
scanId = sf.genScanInstanceGUID()
try:
p = mp.Process(target=SpiderFootScanner,
args=(scanname, scanId, scantarget, targetType,
modlist, cfg))
p.daemon = True
p.start()
except BaseException as e:
print("[-] Scan [%s] failed: %s" % (scanId, e))
return self.error("Scan [%s] failed: %s" % (scanId, e))
# Wait until the scan has initialized
while dbh.scanInstanceGet(scanId) is None:
print("[info] Waiting for the scan to initialize...")
time.sleep(1)
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(rerunscans=True,
docroot=self.docroot,
pageid="SCANLIST")
rerunscanmulti.exposed = True
def newscan(self):
"""
Configure a new scan
"""
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(pageid='NEWSCAN',
types=types,
docroot=self.docroot,
modules=self.config['__modules__'],
scanname="",
selectedmods="",
scantarget="")
newscan.exposed = True
def clonescan(self, id):
"""
Clone an existing scan (pre-selected options in the newscan page)
Args:
id (str): scan ID to clone
"""
sf = SpiderFoot(self.config)
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
info = dbh.scanInstanceGet(id)
if not info:
return self.error("Invalid scan ID.")
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if scanname == "" or scantarget == "" or len(scanconfig) == 0:
return self.error("Something went wrong internally.")
targetType = sf.targetType(scantarget)
if targetType is None:
# It must be a name, so wrap quotes around it
scantarget = """ + scantarget + """
modlist = scanconfig['_modulesenabled'].split(',')
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(pageid='NEWSCAN',
types=types,
docroot=self.docroot,
modules=self.config['__modules__'],
selectedmods=modlist,
scanname=str(scanname),
scantarget=str(scantarget))
clonescan.exposed = True
def index(self):
"""
Main page listing scans available
"""
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(pageid='SCANLIST', docroot=self.docroot)
index.exposed = True
def scaninfo(self, id):
"""Information about a selected scan
Args:
id (str): scan id
"""
dbh = SpiderFootDb(self.config)
res = dbh.scanInstanceGet(id)
if res is None:
return self.error("Scan ID not found.")
templ = Template(filename='dyn/scaninfo.tmpl',
lookup=self.lookup,
input_encoding='utf-8')
return templ.render(id=id,
name=html.escape(res[0]),
status=res[5],
docroot=self.docroot,
pageid="SCANLIST")
scaninfo.exposed = True
def opts(self, updated=None):
"""Settings
Args:
updated: TBD
"""
templ = Template(filename='dyn/opts.tmpl', lookup=self.lookup)
self.token = random.SystemRandom().randint(0, 99999999)
return templ.render(opts=self.config,
pageid='SETTINGS',
token=self.token,
updated=updated,
docroot=self.docroot)
opts.exposed = True
def optsexport(self, pattern):
"""Export configuration
Args:
pattern: TBD
"""
sf = SpiderFoot(self.config)
conf = sf.configSerialize(self.config)
content = ""
for opt in sorted(conf):
if ":_" in opt or opt.startswith("_"):
continue
if pattern:
if pattern in opt:
content += "%s=%s\n" % (opt, conf[opt])
else:
content += "%s=%s\n" % (opt, conf[opt])
cherrypy.response.headers[
'Content-Disposition'] = 'attachment; filename="SpiderFoot.cfg"'
cherrypy.response.headers['Content-Type'] = "text/plain"
return content
optsexport.exposed = True
def optsraw(self):
"""Settings"""
ret = dict()
self.token = random.SystemRandom().randint(0, 99999999)
for opt in self.config:
if not opt.startswith('__'):
ret["global." + opt] = self.config[opt]
continue
if opt == '__modules__':
for mod in sorted(self.config['__modules__'].keys()):
for mo in sorted(
self.config['__modules__'][mod]['opts'].keys()):
if mo.startswith("_"):
continue
ret["module." + mod + "." +
mo] = self.config['__modules__'][mod]['opts'][mo]
return json.dumps(['SUCCESS', {'token': self.token, 'data': ret}])
optsraw.exposed = True
def error(self, message):
"""Generic error, but not exposed as not called directly
Args:
message (str): error message
"""
templ = Template(filename='dyn/error.tmpl', lookup=self.lookup)
return templ.render(message=message, docroot=self.docroot)
def scandelete(self, id, confirm=None, raw=False):
"""Delete a scan
Args:
id (str): scan ID
confirm: TBD
raw: TBD
"""
dbh = SpiderFootDb(self.config)
res = dbh.scanInstanceGet(id)
if res is None:
if raw:
return json.dumps(["ERROR", "Scan ID not found."])
return self.error("Scan ID not found.")
if confirm:
dbh.scanInstanceDelete(id)
if raw:
return json.dumps(["SUCCESS", ""])
raise cherrypy.HTTPRedirect("/")
templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup)
return templ.render(id=id,
name=str(res[0]),
names=list(),
ids=list(),
pageid="SCANLIST",
docroot=self.docroot)
scandelete.exposed = True
def scandeletemulti(self, ids, confirm=None):
"""Delete a scan
Args:
ids (str): comma separated list of scan IDs
confirm: TBD
"""
dbh = SpiderFootDb(self.config)
names = list()
for id in ids.split(','):
res = dbh.scanInstanceGet(id)
names.append(str(res[0]))
if res is None:
return self.error("Scan ID not found (" + id + ").")
if res[5] in ["RUNNING", "STARTING", "STARTED"]:
return self.error("You cannot delete running scans.")
if confirm:
for id in ids.split(','):
dbh.scanInstanceDelete(id)
raise cherrypy.HTTPRedirect("/")
templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup)
return templ.render(id=None,
name=None,
ids=ids.split(','),
names=names,
pageid="SCANLIST",
docroot=self.docroot)
scandeletemulti.exposed = True
def savesettings(self, allopts, token, configFile=None):
"""Save settings, also used to completely reset them to default
Args:
allopts: TBD
token: TBD
configFile: TBD
"""
if str(token) != str(self.token):
return self.error("Invalid token (" + str(self.token) + ").")
if configFile: # configFile seems to get set even if a file isn't uploaded
if configFile.file:
contents = configFile.file.read()
if type(contents) == bytes:
contents = contents.decode("utf-8")
try:
tmp = dict()
for line in contents.split("\n"):
if "=" not in line:
continue
opt_array = line.strip().split("=")
if len(opt_array) == 1:
opt_array[1] = ""
tmp[opt_array[0]] = '='.join(opt_array[1:])
allopts = json.dumps(tmp)
except BaseException as e:
return self.error(
"Failed to parse input file. Was it generated from SpiderFoot? (%s)"
% e)
# Reset config to default
if allopts == "RESET":
if self.reset_settings():
raise cherrypy.HTTPRedirect("/opts?updated=1")
else:
return self.error("Failed to reset settings")
# Save settings
try:
dbh = SpiderFootDb(self.config)
useropts = json.loads(allopts)
cleanopts = dict()
for opt in list(useropts.keys()):
cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0]
currentopts = deepcopy(self.config)
# Make a new config where the user options override
# the current system config.
sf = SpiderFoot(self.config)
self.config = sf.configUnserialize(cleanopts, currentopts)
dbh.configSet(sf.configSerialize(self.config))
except Exception as e:
return self.error(
"Processing one or more of your inputs failed: %s" % e)
raise cherrypy.HTTPRedirect("/opts?updated=1")
savesettings.exposed = True
def savesettingsraw(self, allopts, token):
"""Save settings, also used to completely reset them to default
Args:
allopts: TBD
token: TBD
"""
if str(token) != str(self.token):
return json.dumps(
["ERROR", "Invalid token (" + str(self.token) + ")."])
# Reset config to default
if allopts == "RESET":
if self.reset_settings():
return json.dumps(["SUCCESS", ""])
else:
return json.dumps(["ERROR", "Failed to reset settings"])
# Save settings
try:
dbh = SpiderFootDb(self.config)
useropts = json.loads(allopts)
cleanopts = dict()
for opt in list(useropts.keys()):
cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0]
currentopts = deepcopy(self.config)
# Make a new config where the user options override
# the current system config.
sf = SpiderFoot(self.config)
self.config = sf.configUnserialize(cleanopts, currentopts)
dbh.configSet(sf.configSerialize(self.config))
except Exception as e:
return json.dumps([
"ERROR",
"Processing one or more of your inputs failed: %s" % e
])
return json.dumps(["SUCCESS", ""])
savesettingsraw.exposed = True
def reset_settings(self):
"""Reset settings to default"""
try:
dbh = SpiderFootDb(self.config)
dbh.configClear() # Clear it in the DB
self.config = deepcopy(self.defaultConfig) # Clear in memory
except Exception:
return False
return True
def resultsetfp(self, id, resultids, fp):
"""Set a bunch of results (hashes) as false positive
Args:
id (str): scan ID
resultids (str): comma separated list of result IDs
fp (str): 0 or 1
"""
dbh = SpiderFootDb(self.config)
if fp not in ["0", "1"]:
return json.dumps(
["ERROR", "No FP flag set or not set correctly."])
ids = json.loads(resultids)
if not ids:
return json.dumps(["ERROR", "No IDs supplied."])
# Cannot set FPs if a scan is not completed
status = dbh.scanInstanceGet(id)
if not status:
return self.error("Invalid scan ID: %s" % id)
if status[5] not in ["ABORTED", "FINISHED", "ERROR-FAILED"]:
return json.dumps([
"WARNING",
"Scan must be in a finished state when setting False Positives."
])
# Make sure the user doesn't set something as non-FP when the
# parent is set as an FP.
if fp == "0":
data = dbh.scanElementSourcesDirect(id, ids)
for row in data:
if str(row[14]) == "1":
return json.dumps([
"WARNING",
"Cannot unset element %s as False Positive if a parent element is still False Positive."
% id
])
# Set all the children as FPs too.. it's only logical afterall, right?
childs = dbh.scanElementChildrenAll(id, ids)
allIds = ids + childs
ret = dbh.scanResultsUpdateFP(id, allIds, fp)
if ret:
return json.dumps(["SUCCESS", ""])
return json.dumps(["ERROR", "Exception encountered."])
resultsetfp.exposed = True
def eventtypes(self):
"""For the CLI to fetch a list of event types.
Returns:
str: list of event types
"""
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
ret = list()
for r in types:
ret.append([r[1], r[0]])
ret = sorted(ret, key=itemgetter(0))
return json.dumps(ret)
eventtypes.exposed = True
def modules(self):
"""For the CLI to fetch a list of modules.
Returns:
str: list of modules
"""
modinfo = list(self.config['__modules__'].keys())
modinfo.sort()
ret = list()
for m in modinfo:
if "__" in m:
continue
ret.append({
'name': m,
'descr': self.config['__modules__'][m]['descr']
})
return json.dumps(ret)
modules.exposed = True
def ping(self):
"""For the CLI to test connectivity to this server."""
return json.dumps(["SUCCESS", self.config['__version__']])
ping.exposed = True
def query(self, query):
"""For the CLI to run queries against the database."""
data = None
dbh = SpiderFootDb(self.config)
cherrypy.response.headers[
'Content-Type'] = "application/json; charset=utf-8"
if not query:
return json.dumps(["ERROR", "Invalid query."])
if not query.lower().startswith("select"):
return json.dumps([
"ERROR", "Non-SELECTs are unpredictable and not recommended."
])
try:
ret = dbh.dbh.execute(query)
data = ret.fetchall()
columnNames = [c[0] for c in dbh.dbh.description]
data = [dict(zip(columnNames, row)) for row in data]
except BaseException as e:
return json.dumps(["ERROR", str(e)])
return json.dumps(data)
query.exposed = True
def startscan(self,
scanname,
scantarget,
modulelist,
typelist,
usecase,
cli=None):
"""Initiate a scan
Args:
scanname (str): scan name
scantarget (str): scan target
modulelist (str): TBD
typelist (str): TBD
usecase (str): TBD
cli: TBD
"""
# Swap the globalscantable for the database handler
dbh = SpiderFootDb(self.config)
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modlist = list()
sf = SpiderFoot(cfg)
targetType = None
[scanname, scantarget] = self.cleanUserInput([scanname, scantarget])
if scanname == "" or scantarget == "":
if cli:
return json.dumps([
"ERROR",
"Incorrect usage: scan name or target was not specified."
])
return self.error(
"Invalid request: scan name or target was not specified.")
if typelist == "" and modulelist == "" and usecase == "":
if cli:
return json.dumps([
"ERROR", "Incorrect usage: no modules specified for scan."
])
return self.error(
"Invalid request: no modules specified for scan.")
# User selected modules
if modulelist != "":
modlist = modulelist.replace('module_', '').split(',')
# User selected types
if len(modlist) == 0 and typelist != "":
typesx = typelist.replace('type_', '').split(',')
# 1. Find all modules that produce the requested types
modlist = sf.modulesProducing(typesx)
newmods = deepcopy(modlist)
newmodcpy = deepcopy(newmods)
# 2. For each type those modules consume, get modules producing
while len(newmodcpy) > 0:
for etype in sf.eventsToModules(newmodcpy):
xmods = sf.modulesProducing([etype])
for mod in xmods:
if mod not in modlist:
modlist.append(mod)
newmods.append(mod)
newmodcpy = deepcopy(newmods)
newmods = list()
# User selected a use case
if len(modlist) == 0 and usecase != "":
for mod in self.config['__modules__']:
if usecase == 'all' or usecase in self.config['__modules__'][
mod]['cats']:
modlist.append(mod)
# Add our mandatory storage module..
if "sfp__stor_db" not in modlist:
modlist.append("sfp__stor_db")
modlist.sort()
targetType = sf.targetType(scantarget)
if targetType is None:
if cli:
return json.dumps(["ERROR", "Unrecognised target type."])
return self.error(
"Invalid target type. Could not recognize it as a human name, IP address, IP subnet, ASN, domain name or host name."
)
# Delete the stdout module in case it crept in
if "sfp__stor_stdout" in modlist:
modlist.remove("sfp__stor_stdout")
# Start running a new scan
if targetType in ["HUMAN_NAME", "USERNAME"]:
scantarget = scantarget.replace("\"", "")
else:
scantarget = scantarget.lower()
# Start running a new scan
scanId = sf.genScanInstanceGUID()
try:
p = mp.Process(target=SpiderFootScanner,
args=(scanname, scanId, scantarget, targetType,
modlist, cfg))
p.daemon = True
p.start()
except BaseException as e:
print("[-] Scan [%s] failed: %s" % (scanId, e))
return self.error("Scan [%s] failed: %s" % (scanId, e))
# Wait until the scan has initialized
# Check the database for the scan status results
while dbh.scanInstanceGet(scanId) is None:
print("[info] Waiting for the scan to initialize...")
time.sleep(1)
if cli:
return json.dumps(["SUCCESS", scanId])
raise cherrypy.HTTPRedirect(f"/scaninfo?id={scanId}")
startscan.exposed = True
def stopscanmulti(self, ids):
"""Stop a scan
Args:
ids (str): comma separated list of scan IDs
Note:
Unnecessary for now given that only one simultaneous scan is permitted
"""
dbh = SpiderFootDb(self.config)
error = list()
for id in ids.split(","):
scaninfo = dbh.scanInstanceGet(id)
if not scaninfo:
return self.error("Invalid scan ID: %s" % id)
scanname = str(scaninfo[0])
scanstatus = scaninfo[5]
if scanstatus == "FINISHED":
error.append(
"Scan '%s' is in a finished state. <a href='/scandelete?id=%s&confirm=1'>Maybe you want to delete it instead?</a>"
% (scanname, id))
continue
if scanstatus == "ABORTED":
error.append("Scan '" + scanname + "' is already aborted.")
continue
dbh.scanInstanceSet(id, status="ABORT-REQUESTED")
raise cherrypy.HTTPRedirect("/")
stopscanmulti.exposed = True
def stopscan(self, id, cli=None):
"""Stop a scan.
Args:
id (str): scan ID
cli: TBD
"""
dbh = SpiderFootDb(self.config)
scaninfo = dbh.scanInstanceGet(id)
if not scaninfo:
if cli:
return json.dumps(["ERROR", "Invalid scan ID."])
return self.error("Invalid scan ID.")
scanstatus = scaninfo[5]
if scanstatus == "ABORTED":
if cli:
return json.dumps(["ERROR", "Scan already aborted."])
return self.error("The scan is already aborted.")
if not scanstatus == "RUNNING":
if cli:
return json.dumps(
["ERROR", "Scan in an invalid state for stopping."])
return self.error(
"The running scan is currently in the state '%s', please try again later or restart SpiderFoot."
% scanstatus)
dbh.scanInstanceSet(id, status="ABORT-REQUESTED")
if cli:
return json.dumps(["SUCCESS", ""])
raise cherrypy.HTTPRedirect("/")
stopscan.exposed = True
#
# DATA PROVIDERS
#
def scanlog(self, id, limit=None, rowId=None, reverse=None):
"""Scan log data
Args:
id: TBD
limit: TBD
rowId: TBD
reverse: TBD
"""
dbh = SpiderFootDb(self.config)
retdata = []
try:
data = dbh.scanLogs(id, limit, rowId, reverse)
except Exception:
return json.dumps(retdata)
for row in data:
generated = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0] / 1000))
retdata.append(
[generated, row[1], row[2],
html.escape(row[3]), row[4]])
return json.dumps(retdata)
scanlog.exposed = True
def scanerrors(self, id, limit=None):
"""Scan error data
Args:
id (str): scan ID
limit: TBD
"""
dbh = SpiderFootDb(self.config)
retdata = []
try:
data = dbh.scanErrors(id, limit)
except Exception:
return json.dumps(retdata)
for row in data:
generated = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0] / 1000))
retdata.append([generated, row[1], html.escape(str(row[2]))])
return json.dumps(retdata)
scanerrors.exposed = True
def scanlist(self):
"""Produce a list of scans"""
dbh = SpiderFootDb(self.config)
data = dbh.scanInstanceList()
retdata = []
for row in data:
created = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[3]))
if row[4] == 0:
started = "Not yet"
else:
started = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[4]))
if row[5] == 0:
finished = "Not yet"
else:
finished = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[5]))
retdata.append([
row[0], row[1], row[2], created, started, finished, row[6],
row[7]
])
return json.dumps(retdata)
scanlist.exposed = True
def scanstatus(self, id):
"""Basic information about a scan
Args:
id (str): scan ID
"""
dbh = SpiderFootDb(self.config)
data = dbh.scanInstanceGet(id)
if not data:
return json.dumps([])
created = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[2]))
started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[3]))
ended = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[4]))
retdata = [data[0], data[1], created, started, ended, data[5]]
return json.dumps(retdata)
scanstatus.exposed = True
def scansummary(self, id, by):
"""Summary of scan results
Args:
id (str): scan ID
by: TBD
"""
retdata = []
dbh = SpiderFootDb(self.config)
try:
scandata = dbh.scanResultSummary(id, by)
except Exception:
return json.dumps(retdata)
try:
statusdata = dbh.scanInstanceGet(id)
except Exception:
return json.dumps(retdata)
for row in scandata:
if row[0] == "ROOT":
continue
lastseen = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[2]))
retdata.append(
[row[0], row[1], lastseen, row[3], row[4], statusdata[5]])
return json.dumps(retdata)
scansummary.exposed = True
def scaneventresults(self, id, eventType, filterfp=False):
"""Event results for a scan
Args:
id (str): scan ID
eventType (str): filter by event type
filterfp: TBD
"""
retdata = []
dbh = SpiderFootDb(self.config)
try:
data = dbh.scanResultEvent(id, eventType, filterfp)
except Exception:
return json.dumps(retdata)
for row in data:
lastseen = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(row[0]))
escapeddata = html.escape(row[1])
escapedsrc = html.escape(row[2])
retdata.append([
lastseen, escapeddata, escapedsrc, row[3], row[5], row[6],
row[7], row[8], row[13], row[14], row[4]
])
return json.dumps(retdata, ensure_ascii=False)
scaneventresults.exposed = True
def scaneventresultsunique(self, id, eventType, filterfp=False):
"""Unique event results for a scan
Args:
id (str): scan ID
eventType (str): filter by event type
filterfp: TBD
"""
dbh = SpiderFootDb(self.config)
retdata = []
try:
data = dbh.scanResultEventUnique(id, eventType, filterfp)
except Exception:
return json.dumps(retdata)
for row in data:
escaped = html.escape(row[0])
retdata.append([escaped, row[1], row[2]])
return json.dumps(retdata, ensure_ascii=False)
scaneventresultsunique.exposed = True
def search(self, id=None, eventType=None, value=None):
"""Search
Args:
id: TBD
eventType (str): filter by event type
value: TBD
"""
try:
data = self.searchBase(id, eventType, value)
except Exception:
return json.dumps([])
return json.dumps(data, ensure_ascii=False)
search.exposed = True
def scanhistory(self, id):
"""Historical data for a scan.
Args:
id (str): scan ID
"""
dbh = SpiderFootDb(self.config)
try:
data = dbh.scanResultHistory(id)
except Exception:
return json.dumps([])
return json.dumps(data, ensure_ascii=False)
scanhistory.exposed = True
def scanelementtypediscovery(self, id, eventType):
"""scan element type discovery
Args:
id: TBD
eventType (str): filter by event type
"""
sf = SpiderFoot(self.config)
dbh = SpiderFootDb(self.config)
pc = dict()
datamap = dict()
# Get the events we will be tracing back from
leafSet = dbh.scanResultEvent(id, eventType)
[datamap, pc] = dbh.scanElementSourcesAll(id, leafSet)
# Delete the ROOT key as it adds no value from a viz perspective
del pc['ROOT']
retdata = dict()
retdata['tree'] = sf.dataParentChildToTree(pc)
retdata['data'] = datamap
return json.dumps(retdata, ensure_ascii=False)
scanelementtypediscovery.exposed = True
from flask import Flask, request
app = Flask(__name__)
from mako.template import Template as MakoTemplates
from mako.lookup import TemplateLookup
from jinja2 import Environment as Jinja2Environment
import tornado.template
import random
import string
import time
mylookup = TemplateLookup(directories=['/tpl'])
Jinja2Env = Jinja2Environment(line_statement_prefix='#')
def shutdown_server():
func = request.environ.get('werkzeug.server.shutdown')
if func is None:
raise RuntimeError('Not running with the Werkzeug Server')
func()
def randomword(length=8):
return ''.join(random.choice(string.lowercase) for i in range(length))
@app.route("/reflect/<engine>")
def reflect(engine):
template = request.values.get('tpl')
if not template:
def __init__(self,
short_name,
full_name,
owners,
categories,
added_default=None,
added_mandatory=None,
node_settings_model=None,
user_settings_model=None,
include_js=None,
include_css=None,
widget_help=None,
views=None,
configs=None,
models=None,
has_hgrid_files=False,
get_hgrid_data=None,
max_file_size=None,
high_max_file_size=None,
accept_extensions=True,
node_settings_template=None,
user_settings_template=None,
**kwargs):
self.models = models
self.settings_models = {}
if node_settings_model:
node_settings_model.config = self
self.settings_models['node'] = node_settings_model
if user_settings_model:
user_settings_model.config = self
self.settings_models['user'] = user_settings_model
self.short_name = short_name
self.full_name = full_name
self.owners = owners
self.categories = categories
self.added_default = added_default or []
self.added_mandatory = added_mandatory or []
if set(self.added_mandatory).difference(self.added_default):
raise ValueError('All mandatory targets must also be defaults.')
self.include_js = self._include_to_static(include_js or {})
self.include_css = self._include_to_static(include_css or {})
self.widget_help = widget_help
self.views = views or []
self.configs = configs or []
self.has_hgrid_files = has_hgrid_files
# WARNING: get_hgrid_data can return None if the addon is added but has no credentials.
self.get_hgrid_data = get_hgrid_data # if has_hgrid_files and not get_hgrid_data rubeus.make_dummy()
self.max_file_size = max_file_size
self.high_max_file_size = high_max_file_size
self.accept_extensions = accept_extensions
# Build template lookup
template_path = os.path.join('website', 'addons', short_name,
'templates')
if os.path.exists(template_path):
self.template_lookup = TemplateLookup(directories=[
template_path,
settings.TEMPLATES_PATH,
])
else:
self.template_lookup = None
# Provide the path the the user_settings template
self.user_settings_template = user_settings_template
addon_user_settings_path = os.path.join(
template_path, '{}_user_settings.mako'.format(self.short_name))
if user_settings_template:
# If USER_SETTINGS_TEMPLATE is defined, use that path.
self.user_settings_template = user_settings_template
elif os.path.exists(addon_user_settings_path):
# An implicit template exists
self.user_settings_template = os.path.join(
os.path.pardir,
'addons',
self.short_name,
'templates',
'{}_user_settings.mako'.format(self.short_name),
)
else:
# Use the default template (for OAuth addons)
self.user_settings_template = os.path.join(
'project',
'addon',
'user_settings_default.mako',
)
self.node_settings_template = node_settings_template
def __init__(path):
self._lookup = TemplateLookup(directories=[path],
module_directory='mako_modules')
help="The ID of the entities descriptor")
parser.add_argument('-k',
dest='keyfile',
help="A file with a key to sign the metadata with")
parser.add_argument('-n', dest='name')
parser.add_argument('-s',
dest='sign',
action='store_true',
help="sign the metadata")
parser.add_argument('-m', dest='mako_root', default="./")
parser.add_argument(dest="config")
args = parser.parse_args()
_rot = args.mako_root
LOOKUP = TemplateLookup(directories=[_rot + 'templates', _rot + 'htdocs'],
module_directory=_rot + 'modules',
input_encoding='utf-8',
output_encoding='utf-8')
PORT = 8088
AUTHN_BROKER = AuthnBroker()
AUTHN_BROKER.add(authn_context_class_ref(PASSWORD),
username_password_authn, 10,
"http://%s" % socket.gethostname())
AUTHN_BROKER.add(authn_context_class_ref(UNSPECIFIED), "", 0,
"http://%s" % socket.gethostname())
IDP = server.Server(args.config, cache=Cache())
IDP.ticket = {}
SRV = make_server('', PORT, application)
def main(name):
template_lookup = TemplateLookup(directories=[name])
template = template_lookup.get_template('welcome.html')
return lambda ctx: template.render(**ctx)
