def __init__(self, generated_root): from mako.lookup import TemplateLookup self.generated_root = generated_root self.template_lookup = TemplateLookup(directories=['templates'])
from mako.lookup import TemplateLookup import furl import requests from framework.exceptions import PermissionsError from framework.mongo import StoredObject from framework.routing import process_rules from framework.guid.model import GuidStoredObject from website import settings from website.addons.base import exceptions from website.addons.base import serializer from website.project.model import Node lookup = TemplateLookup(directories=[settings.TEMPLATES_PATH]) STATUS_EXCEPTIONS = { 410: exceptions.FileDeletedError, 404: exceptions.FileDoesntExistError } def _is_image(filename): mtype, _ = mimetypes.guess_type(filename) return mtype and mtype.startswith('image') class AddonConfig(object): def __init__(self, short_name,
def __init__(self, view_search_folder): self.lookup_o = TemplateLookup(directories=view_search_folder, input_encoding='utf-8')
class SpiderFootWebUi: lookup = TemplateLookup(directories=[''], output_encoding='utf-8') defaultConfig = dict() config = dict() token = None docroot = '' def __init__(self, config): self.defaultConfig = deepcopy(config) dbh = SpiderFootDb(self.defaultConfig) # 'config' supplied will be the defaults, let's supplement them # now with any configuration which may have previously been # saved. sf = SpiderFoot(self.defaultConfig) self.config = sf.configUnserialize(dbh.configGet(), self.defaultConfig) if self.config['__webaddr'] == "0.0.0.0": addr = "<IP of this host>" else: addr = self.config['__webaddr'] self.docroot = self.config['__docroot'].rstrip('/') print "" print "" print "*************************************************************" print " Use SpiderFoot by starting your web browser of choice and " print " browse to http://" + addr + ":" + str( self.config['__webport']) + self.docroot print "*************************************************************" print "" print "" # Sanitize user input def cleanUserInput(self, inputList): ret = list() for item in inputList: c = cgi.escape(item, True) c = c.replace('\'', '"') # We don't actually want & translated to & c = c.replace("&", "&") ret.append(c) return ret def searchBase(self, id=None, eventType=None, value=None): regex = "" if [id, eventType, value].count('') == 2 or \ [id, eventType, value].count(None) == 2: return None if value.startswith("/") and value.endswith("/"): regex = value[1:len(value) - 1] value = "" value = value.replace('*', '%') if value in [None, ""] and regex in [None, ""]: value = "%" regex = "" dbh = SpiderFootDb(self.config) criteria = { 'scan_id': None if id == '' else id, 'type': None if eventType == '' else eventType, 'value': None if value == '' else value, 'regex': None if regex == '' else regex } data = dbh.search(criteria) retdata = [] for row in data: lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0])) escapeddata = cgi.escape(row[1]) escapedsrc = cgi.escape(row[2]) retdata.append([ lastseen, escapeddata, escapedsrc, row[3], row[5], row[6], row[7], row[8], row[10], row[11], row[4], row[13], row[14] ]) return retdata # # USER INTERFACE PAGES # # Get result data in CSV format def scaneventresultexport(self, id, type, dialect="excel"): dbh = SpiderFootDb(self.config) data = dbh.scanResultEvent(id, type) fileobj = StringIO() parser = csv.writer(fileobj, dialect=dialect) parser.writerow(["Updated", "Type", "Module", "Source", "F/P", "Data"]) for row in data: if row[4] == "ROOT": continue lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0])) datafield = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "") parser.writerow([ lastseen, str(row[4]), str(row[3]), str(row[2]), row[13], datafield ]) cherrypy.response.headers[ 'Content-Disposition'] = "attachment; filename=SpiderFoot.csv" cherrypy.response.headers['Content-Type'] = "application/csv" cherrypy.response.headers['Pragma'] = "no-cache" return fileobj.getvalue() scaneventresultexport.exposed = True # Get result data in CSV format for multiple scans def scaneventresultexportmulti(self, ids, dialect="excel"): dbh = SpiderFootDb(self.config) scaninfo = dict() data = list() for id in ids.split(','): scaninfo[id] = dbh.scanInstanceGet(id) data = data + dbh.scanResultEvent(id) fileobj = StringIO() parser = csv.writer(fileobj, dialect=dialect) parser.writerow([ "Scan Name", "Updated", "Type", "Module", "Source", "F/P", "Data" ]) for row in data: if row[4] == "ROOT": continue lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0])) datafield = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "") parser.writerow([ scaninfo[row[12]][0], lastseen, str(row[4]), str(row[3]), str(row[2]), row[13], datafield ]) cherrypy.response.headers[ 'Content-Disposition'] = "attachment; filename=SpiderFoot.csv" cherrypy.response.headers['Content-Type'] = "application/csv" cherrypy.response.headers['Pragma'] = "no-cache" return fileobj.getvalue() scaneventresultexportmulti.exposed = True # Get search result data in CSV format def scansearchresultexport(self, id, eventType=None, value=None, dialect="excel"): data = self.searchBase(id, eventType, value) fileobj = StringIO() parser = csv.writer(fileobj, dialect=dialect) parser.writerow(["Updated", "Type", "Module", "Source", "F/P", "Data"]) for row in data: if row[10] == "ROOT": continue datafield = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "") parser.writerow([ row[0], str(row[10]), str(row[3]), str(row[2]), row[13], datafield ]) cherrypy.response.headers[ 'Content-Disposition'] = "attachment; filename=SpiderFoot.csv" cherrypy.response.headers['Content-Type'] = "application/csv" cherrypy.response.headers['Pragma'] = "no-cache" return fileobj.getvalue() scansearchresultexport.exposed = True # Export entities from scan results for visualising def scanviz(self, id, gexf="0"): types = list() dbh = SpiderFootDb(self.config) sf = SpiderFoot(self.config) data = dbh.scanResultEvent(id, filterFp=True) scan = dbh.scanInstanceGet(id) root = scan[1] if gexf != "0": cherrypy.response.headers[ 'Content-Disposition'] = "attachment; filename=SpiderFoot.gexf" cherrypy.response.headers['Content-Type'] = "application/gexf" cherrypy.response.headers['Pragma'] = "no-cache" return sf.buildGraphGexf([root], "SpiderFoot Export", data) else: return sf.buildGraphJson([root], data) scanviz.exposed = True # Export entities results from multiple scans in GEXF format def scanvizmulti(self, ids, gexf="1"): types = list() dbh = SpiderFootDb(self.config) sf = SpiderFoot(self.config) data = list() roots = list() for id in ids.split(','): data = data + dbh.scanResultEvent(id, filterFp=True) roots.append(dbh.scanInstanceGet(id)[1]) if gexf != "0": cherrypy.response.headers[ 'Content-Disposition'] = "attachment; filename=SpiderFoot.gexf" cherrypy.response.headers['Content-Type'] = "application/gexf" cherrypy.response.headers['Pragma'] = "no-cache" return sf.buildGraphGexf(roots, "SpiderFoot Export", data) else: # Not implemented yet return None scanvizmulti.exposed = True # Configuration used for a scan def scanopts(self, id): ret = dict() dbh = SpiderFootDb(self.config) ret['config'] = dbh.scanConfigGet(id) ret['configdesc'] = dict() for key in ret['config'].keys(): if ':' not in key: ret['configdesc'][key] = self.config['__globaloptdescs__'][key] else: [modName, modOpt] = key.split(':') if modName not in self.config['__modules__'].keys(): continue if modOpt not in self.config['__modules__'][modName][ 'optdescs'].keys(): continue ret['configdesc'][key] = self.config['__modules__'][modName][ 'optdescs'][modOpt] sf = SpiderFoot(self.config) meta = dbh.scanInstanceGet(id) if not meta: return json.dumps([]) if meta[3] != 0: started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(meta[3])) else: started = "Not yet" if meta[4] != 0: finished = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(meta[4])) else: finished = "Not yet" ret['meta'] = [meta[0], meta[1], meta[2], started, finished, meta[5]] return json.dumps(ret) scanopts.exposed = True def rerunscan(self, id): # Snapshot the current configuration to be used by the scan cfg = deepcopy(self.config) modopts = dict() # Not used yet as module options are set globally modlist = list() sf = SpiderFoot(cfg) dbh = SpiderFootDb(cfg) info = dbh.scanInstanceGet(id) scanconfig = dbh.scanConfigGet(id) scanname = info[0] scantarget = info[1] targetType = None if len(scanconfig) == 0: return self.error("Something went wrong internally.") modlist = scanconfig['_modulesenabled'].split(',') targetType = sf.targetType(scantarget) if targetType == None: # Should never be triggered for a re-run scan.. return self.error("Invalid target type. Could not recognize it as " + \ "an IP address, IP subnet, domain name or host name.") # Start running a new scan newId = sf.genScanInstanceGUID(scanname) t = SpiderFootScanner(scanname, scantarget.lower(), targetType, newId, modlist, cfg, modopts) t.start() # Wait until the scan has initialized while globalScanStatus.getStatus(newId) == None: print "[info] Waiting for the scan to initialize..." time.sleep(1) templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup) return templ.render(id=newId, name=scanname, docroot=self.docroot, status=globalScanStatus.getStatus(newId), pageid="SCANLIST") rerunscan.exposed = True def rerunscanmulti(self, ids): # Snapshot the current configuration to be used by the scan cfg = deepcopy(self.config) modopts = dict() # Not used yet as module options are set globally modlist = list() sf = SpiderFoot(cfg) dbh = SpiderFootDb(cfg) for id in ids.split(","): info = dbh.scanInstanceGet(id) scanconfig = dbh.scanConfigGet(id) scanname = info[0] scantarget = info[1] targetType = None if len(scanconfig) == 0: return self.error("Something went wrong internally.") modlist = scanconfig['_modulesenabled'].split(',') targetType = sf.targetType(scantarget) if targetType == None: # Should never be triggered for a re-run scan.. return self.error("Invalid target type. Could not recognize it as " + \ "an IP address, IP subnet, domain name or host name.") # Start running a new scan newId = sf.genScanInstanceGUID(scanname) t = SpiderFootScanner(scanname, scantarget.lower(), targetType, newId, modlist, cfg, modopts) t.start() # Wait until the scan has initialized while globalScanStatus.getStatus(newId) == None: print "[info] Waiting for the scan to initialize..." time.sleep(1) templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup) return templ.render(rerunscans=True, docroot=self.docroot, pageid="SCANLIST") rerunscanmulti.exposed = True # Configure a new scan def newscan(self): dbh = SpiderFootDb(self.config) types = dbh.eventTypes() templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup) return templ.render(pageid='NEWSCAN', types=types, docroot=self.docroot, modules=self.config['__modules__'], scanname="", selectedmods="", scantarget="") newscan.exposed = True # Clone an existing scan (pre-selected options in the newscan page) def clonescan(self, id): dbh = SpiderFootDb(self.config) types = dbh.eventTypes() info = dbh.scanInstanceGet(id) scanconfig = dbh.scanConfigGet(id) scanname = info[0] scantarget = info[1] targetType = None if scanname == "" or scantarget == "" or len(scanconfig) == 0: return self.error("Something went wrong internally.") modlist = scanconfig['_modulesenabled'].split(',') templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup) return templ.render(pageid='NEWSCAN', types=types, docroot=self.docroot, modules=self.config['__modules__'], selectedmods=modlist, scanname=scanname, scantarget=scantarget) clonescan.exposed = True # Main page listing scans available def index(self): # Look for referenced templates in the current directory only templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup) return templ.render(pageid='SCANLIST', docroot=self.docroot) index.exposed = True # Information about a selected scan def scaninfo(self, id): dbh = SpiderFootDb(self.config) res = dbh.scanInstanceGet(id) if res is None: return self.error("Scan ID not found.") scanName = SpiderFoot.znEncode(res[0]) target = res[1] templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup, input_encoding='utf-8') # templ = Template(filename='dyn/scaninfo.tmpl', default_filters=['decode.utf8'], input_encoding='utf-8', output_encoding='utf-8') return templ.render(id=id, name=target, status=res[5], docroot=self.docroot, pageid="SCANLIST") scaninfo.exposed = True # Settings def opts(self): templ = Template(filename='dyn/opts.tmpl', lookup=self.lookup) self.token = random.randint(0, 99999999) return templ.render(opts=self.config, pageid='SETTINGS', token=self.token, docroot=self.docroot) opts.exposed = True # Settings def optsraw(self): ret = dict() self.token = random.randint(0, 99999999) for opt in self.config: if opt.startswith('__'): if opt == '__modules__': for mod in sorted(self.config['__modules__'].keys()): for mo in sorted(self.config['__modules__'][mod] ['opts'].keys()): if mo.startswith("_"): continue ret["module." + mod + "." + mo] = self.config[ '__modules__'][mod]['opts'][mo] continue ret["global." + opt] = self.config[opt] return json.dumps(['SUCCESS', {'token': self.token, 'data': ret}]) optsraw.exposed = True # Generic error, but not exposed as not called directly def error(self, message): templ = Template(filename='dyn/error.tmpl', lookup=self.lookup) return templ.render(message=message, docroot=self.docroot) # Delete a scan def scandelete(self, id, confirm=None, raw=False): dbh = SpiderFootDb(self.config) res = dbh.scanInstanceGet(id) if res is None: if not raw: return self.error("Scan ID not found.") else: return json.dumps(["ERROR", "Scan ID not found."]) if confirm is not None: dbh.scanInstanceDelete(id) if not raw: raise cherrypy.HTTPRedirect("/") else: return json.dumps(["SUCCESS", ""]) else: templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup) return templ.render(id=id, name=res[0], names=list(), ids=list(), pageid="SCANLIST", docroot=self.docroot) scandelete.exposed = True # Delete a scan def scandeletemulti(self, ids, confirm=None): dbh = SpiderFootDb(self.config) names = list() for id in ids.split(','): res = dbh.scanInstanceGet(id) names.append(res[0]) if res is None: return self.error("Scan ID not found (" + id + ").") if res[5] in ["RUNNING", "STARTING", "STARTED"]: return self.error("You cannot delete running scans.") if confirm is not None: for id in ids.split(','): dbh.scanInstanceDelete(id) raise cherrypy.HTTPRedirect("/") else: templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup) return templ.render(id=None, name=None, ids=ids.split(','), names=names, pageid="SCANLIST", docroot=self.docroot) scandeletemulti.exposed = True # Save settings, also used to completely reset them to default def savesettings(self, allopts, token): # if str(token) != str(self.token): # return self.error("Invalid token (" + str(self.token) + ").") try: dbh = SpiderFootDb(self.config) # Reset config to default if allopts == "RESET": dbh.configClear() # Clear it in the DB self.config = deepcopy(self.defaultConfig) # Clear in memory else: useropts = json.loads(allopts) cleanopts = dict() for opt in useropts.keys(): cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0] currentopts = deepcopy(self.config) # Make a new config where the user options override # the current system config. sf = SpiderFoot(self.config) self.config = sf.configUnserialize(cleanopts, currentopts) dbh.configSet(sf.configSerialize(currentopts)) except Exception as e: return self.error( "Processing one or more of your inputs failed: " + str(e)) templ = Template(filename='dyn/opts.tmpl', lookup=self.lookup) self.token = random.randint(0, 99999999) return templ.render(opts=self.config, pageid='SETTINGS', updated=True, docroot=self.docroot, token=self.token) savesettings.exposed = True # Save settings, also used to completely reset them to default def savesettingsraw(self, allopts, token): if str(token) != str(self.token): return json.dumps( ["ERROR", "Invalid token (" + str(self.token) + ")."]) try: dbh = SpiderFootDb(self.config) # Reset config to default if allopts == "RESET": dbh.configClear() # Clear it in the DB self.config = deepcopy(self.defaultConfig) # Clear in memory else: useropts = json.loads(allopts) cleanopts = dict() for opt in useropts.keys(): cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0] currentopts = deepcopy(self.config) # Make a new config where the user options override # the current system config. sf = SpiderFoot(self.config) self.config = sf.configUnserialize(cleanopts, currentopts) dbh.configSet(sf.configSerialize(currentopts)) except Exception as e: return json.dumps([ "ERROR", "Processing one or more of your inputs failed: " + str(e) ]) return json.dumps(["SUCCESS", ""]) savesettingsraw.exposed = True # Set a bunch of results (hashes) as false positive def resultsetfp(self, id, resultids, fp): dbh = SpiderFootDb(self.config) if fp not in ["0", "1"]: return json.dumps( ["ERROR", "No FP flag set or not set correctly."]) ids = json.loads(resultids) if not ids: return json.dumps(["ERROR", "No IDs supplied."]) # Cannot set FPs if a scan is not completed status = dbh.scanInstanceGet(id) if status[5] not in ["ABORTED", "FINISHED", "ERROR-FAILED"]: return json.dumps(["WARNING", "Scan must be in a finished state when " + \ "setting False Positives."]) # Make sure the user doesn't set something as non-FP when the # parent is set as an FP. if fp == "0": data = dbh.scanElementSourcesDirect(id, ids) for row in data: if str(row[14]) == "1": return json.dumps(["WARNING", "You cannot unset an element as False Positive " + \ "if a parent element is still False Positive."]) # Set all the children as FPs too.. it's only logical afterall, right? childs = dbh.scanElementChildrenAll(id, ids) allIds = ids + childs ret = dbh.scanResultsUpdateFP(id, allIds, fp) if not ret: return json.dumps(["ERROR", "Exception encountered."]) else: return json.dumps(["SUCCESS", ""]) resultsetfp.exposed = True # For the CLI to fetch a list of event types. def eventtypes(self): dbh = SpiderFootDb(self.config) types = dbh.eventTypes() ret = list() for r in types: ret.append([r[1], r[0]]) ret = sorted(ret, key=itemgetter(0)) return json.dumps(ret) eventtypes.exposed = True # For the CLI to fetch a list of modules. def modules(self): modinfo = self.config['__modules__'].keys() modinfo.sort() ret = list() for m in modinfo: if "__" in m: continue ret.append({ 'name': m, 'descr': self.config['__modules__'][m]['descr'] }) return json.dumps(ret) modules.exposed = True # For the CLI to test connectivity to this server. def ping(self): return json.dumps(["SUCCESS", self.config['__version__']]) ping.exposed = True # For the CLI to run queries against the database. def query(self, query): data = None dbh = SpiderFootDb(self.config) if not query.lower().startswith("select"): return json.dumps([ "ERROR", "Non-SELECTs are unpredictable and not recommended." ]) try: ret = dbh.dbh.execute(query) data = ret.fetchall() except BaseException as e: return json.dumps(["ERROR", str(e)]) return json.dumps(data) query.exposed = True # Initiate a scan def startscan(self, scanname, scantarget, modulelist, typelist, usecase, cli=None): global globalScanStatus # Snapshot the current configuration to be used by the scan cfg = deepcopy(self.config) modopts = dict() # Not used yet as module options are set globally modlist = list() sf = SpiderFoot(cfg) dbh = SpiderFootDb(cfg) types = dbh.eventTypes() targetType = None scanname = SpiderFoot.znEncode(scanname) [scanname, scantarget] = self.cleanUserInput([scanname, scantarget]) if scanname == "" or scantarget == "": if not cli: return self.error("Form incomplete.") else: return json.dumps(["ERROR", "Incorrect usage."]) if typelist == "" and modulelist == "" and usecase == "": if not cli: return self.error("Form incomplete.") else: return json.dumps(["ERROR", "Incorrect usage."]) # User selected modules if modulelist != "": modlist = modulelist.replace('module_', '').split(',') # User selected types if len(modlist) == 0 and typelist != "": typesx = typelist.replace('type_', '').split(',') # 1. Find all modules that produce the requested types modlist = sf.modulesProducing(typesx) newmods = deepcopy(modlist) newmodcpy = deepcopy(newmods) # 2. For each type those modules consume, get modules producing while len(newmodcpy) > 0: for etype in sf.eventsToModules(newmodcpy): xmods = sf.modulesProducing([etype]) for mod in xmods: if mod not in modlist: modlist.append(mod) newmods.append(mod) newmodcpy = deepcopy(newmods) newmods = list() # User selected a use case if len(modlist) == 0 and usecase != "": for mod in self.config['__modules__']: if usecase == 'all' or usecase in self.config['__modules__'][ mod]['cats']: modlist.append(mod) # Add our mandatory storage module.. if "sfp__stor_db" not in modlist: modlist.append("sfp__stor_db") modlist.sort() targetType = sf.targetType(scantarget) if targetType is None: if not cli: return self.error("Invalid target type. Could not recognize it as " + \ "an IP address, IP subnet, domain name or host name.") else: return json.dumps(["ERROR", "Unrecognised target type."]) # Start running a new scan scanId = sf.genScanInstanceGUID(scanname) t = SpiderFootScanner(scanname, scantarget.lower(), targetType, scanId, modlist, cfg, modopts) t.start() # Wait until the scan has initialized while globalScanStatus.getStatus(scanId) is None: print "[info] Waiting for the scan to initialize..." time.sleep(1) scanname = SpiderFoot.znDecode(scanname) if not cli: templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup, input_encoding='utf-8') return templ.render(id=scanId, name=scanname, docroot=self.docroot, status=globalScanStatus.getStatus(scanId), pageid="SCANLIST") else: return json.dumps(["SUCCESS", scanId]) startscan.exposed = True # Stop a scan (id variable is unnecessary for now given that only one simultaneous # scan is permitted.) def stopscanmulti(self, ids): global globalScanStatus # running scans dbh = SpiderFootDb(self.config) error = list() for id in ids.split(","): errState = False scaninfo = dbh.scanInstanceGet(id) if scaninfo is None: return self.error("Invalid scan ID specified.") if globalScanStatus.getStatus( id) == "FINISHED" or scaninfo[5] == "FINISHED": error.append("Scan '" + scaninfo[0] + "' is in a finished state. <a href='/scandelete?id=" + \ id + "&confirm=1'>Maybe you want to delete it instead?</a>") errState = True if not errState and (globalScanStatus.getStatus(id) == "ABORTED" or scaninfo[5] == "ABORTED"): error.append("Scan '" + scaninfo[0] + "' is already aborted.") errState = True if not errState and globalScanStatus.getStatus(id) is None: error.append("Scan '" + scaninfo[0] + "' is not actually running. A data consistency " + \ "error for this scan probably exists. <a href='/scandelete?id=" + \ id + "&confirm=1'>Click here to delete it.</a>") errState = True if not errState: globalScanStatus.setStatus(id, "ABORT-REQUESTED") templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup) return templ.render(pageid='SCANLIST', stoppedscan=True, errors=error, docroot=self.docroot) stopscanmulti.exposed = True # Stop a scan. def stopscan(self, id, cli=None): global globalScanStatus dbh = SpiderFootDb(self.config) scaninfo = dbh.scanInstanceGet(id) if scaninfo is None: if not cli: return self.error("Invalid scan ID.") else: return json.dumps(["ERROR", "Invalid scan ID."]) if globalScanStatus.getStatus(id) is None: if not cli: return self.error("That scan is not actually running. A data consistency " + \ "error for this scan probably exists. <a href='/scandelete?id=" + \ id + "&confirm=1'>Click here to delete it.</a>") else: return json.dumps( ["ERROR", "Scan doesn't appear to be running."]) if globalScanStatus.getStatus(id) == "ABORTED": if not cli: return self.error("The scan is already aborted.") else: return json.dumps(["ERROR", "Scan already aborted."]) if not globalScanStatus.getStatus(id) == "RUNNING": if not cli: return self.error("The running scan is currently in the state '" + \ globalScanStatus.getStatus(id) + "', please try again later or restart " + \ " SpiderFoot.") else: return json.dumps( ["ERROR", "Scan in an invalid state for stopping."]) globalScanStatus.setStatus(id, "ABORT-REQUESTED") if not cli: templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup) return templ.render(pageid='SCANLIST', stoppedscan=True, docroot=self.docroot, errors=list()) else: return json.dumps(["SUCCESS", ""]) stopscan.exposed = True # # DATA PROVIDERS # # Scan log data def scanlog(self, id, limit=None, rowId=None, reverse=None): dbh = SpiderFootDb(self.config) data = dbh.scanLogs(id, limit, rowId, reverse) retdata = [] for row in data: generated = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0] / 1000)) retdata.append([ generated, row[1], row[2], cgi.escape(unicode(row[3], errors='replace')), row[4] ]) return json.dumps(retdata) scanlog.exposed = True # Scan error data def scanerrors(self, id, limit=None): dbh = SpiderFootDb(self.config) data = dbh.scanErrors(id, limit) retdata = [] for row in data: generated = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0] / 1000)) retdata.append([ generated, row[1], cgi.escape(unicode(row[2], errors='replace')) ]) return json.dumps(retdata) scanerrors.exposed = True # Produce a list of scans def scanlist(self): dbh = SpiderFootDb(self.config) data = dbh.scanInstanceList() retdata = [] for row in data: created = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[3])) if row[4] != 0: started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[4])) else: started = "Not yet" if row[5] != 0: finished = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[5])) else: finished = "Not yet" retdata.append([ row[0], row[1], row[2], created, started, finished, row[6], row[7] ]) return json.dumps(retdata) scanlist.exposed = True # Basic information about a scan def scanstatus(self, id): dbh = SpiderFootDb(self.config) data = dbh.scanInstanceGet(id) created = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[2])) started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[3])) ended = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[4])) retdata = [data[0], data[1], created, started, ended, data[5]] return json.dumps(retdata) scanstatus.exposed = True # Summary of scan results def scansummary(self, id, by): dbh = SpiderFootDb(self.config) data = dbh.scanResultSummary(id, by) retdata = [] for row in data: if row[0] == "ROOT": continue lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[2])) retdata.append([row[0], row[1], lastseen, row[3], row[4]]) return json.dumps(retdata) scansummary.exposed = True # Event results for a scan def scaneventresults(self, id, eventType, filterfp=False): dbh = SpiderFootDb(self.config) data = dbh.scanResultEvent(id, eventType, filterfp) retdata = [] for row in data: lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0])) escapeddata = cgi.escape(row[1]) escapedsrc = cgi.escape(row[2]) retdata.append([ lastseen, escapeddata, escapedsrc, row[3], row[5], row[6], row[7], row[8], row[13], row[14], row[4] ]) return json.dumps(retdata, ensure_ascii=False) scaneventresults.exposed = True # Unique event results for a scan def scaneventresultsunique(self, id, eventType, filterfp=False): dbh = SpiderFootDb(self.config) data = dbh.scanResultEventUnique(id, eventType, filterfp) retdata = [] for row in data: escaped = cgi.escape(row[0]) retdata.append([escaped, row[1], row[2]]) return json.dumps(retdata, ensure_ascii=False) scaneventresultsunique.exposed = True # Search def search(self, id=None, eventType=None, value=None): retdata = self.searchBase(id, eventType, value) return json.dumps(retdata, ensure_ascii=False) search.exposed = True # Historical data for the scan, graphs will be rendered in JS def scanhistory(self, id): dbh = SpiderFootDb(self.config) data = dbh.scanResultHistory(id) return json.dumps(data, ensure_ascii=False) scanhistory.exposed = True def scanelementtypediscovery(self, id, eventType): sf = SpiderFoot(self.config) dbh = SpiderFootDb(self.config) pc = dict() datamap = dict() # Get the events we will be tracing back from leafSet = dbh.scanResultEvent(id, eventType) [datamap, pc] = dbh.scanElementSourcesAll(id, leafSet) # Delete the ROOT key as it adds no value from a viz perspective del pc['ROOT'] retdata = dict() retdata['tree'] = sf.dataParentChildToTree(pc) retdata['data'] = datamap return json.dumps(retdata, ensure_ascii=False) scanelementtypediscovery.exposed = True
message = traceback.format_exception(*sys.exc_info()) print(message) logger.exception("%s" % err) resp = ServiceError("%s" % err) return resp(environ, start_response) LOGGER.debug("unknown side: %s" % path) resp = NotFound("Couldn't find the side you asked for!") return resp(environ, start_response) if __name__ == '__main__': root = './' lookup = TemplateLookup(directories=[root + 'Templates', root + 'htdocs'], module_directory=root + 'modules', input_encoding='utf-8', output_encoding='utf-8') usernamePasswords = {"user1": "1", "user2": "2"} passwordEndPointIndex = 0 # what is this, and what does its value mean? # JWKS: JSON Web Key jwksFileName = "static/jwks.json" # parse the parameters parser = argparse.ArgumentParser() parser.add_argument('-c', dest='config') parser.add_argument('-d', dest='debug', action='store_true') args = parser.parse_args()
import importlib import os import sys import codecs from mako.lookup import TemplateLookup MODULE_PATH = os.getenv( 'MODULE_PATH', os.path.realpath(os.path.join(os.path.dirname(__file__), '../modules'))) TEMPLATE_PATH = os.getenv( 'TEMPLATE_PATH', os.path.realpath( os.path.join(os.path.dirname(__file__), 'boilerplate_data'))) VERSION = '1.2' TEMPLATES = TemplateLookup(directories=[TEMPLATE_PATH]) def u8(s): return s.decode('utf-8') def gitconfig(entry): return u8( subprocess.check_output('git config -z --get %s' % entry, shell=True)[:-1]) def write(target, contents): if not os.path.isdir(os.path.dirname(target)): os.makedirs(os.path.dirname(target))
class AggregatePages(object): @cherrypy.expose def index(self): return self.callstacks() templates_dir = os.path.join(os.getcwd(),'static','templates') template_lookup = TemplateLookup(directories=[templates_dir,]) @cherrypy.expose def callstacks(self, id=None, **kwargs): table_kwargs, filter_kwargs = parse_kwargs(kwargs) if 'id' in filter_kwargs: filter_kwargs.pop('id') for k in filter_kwargs: filter_kwargs[k] = str(filter_kwargs[k]) if id: call_stack, total, filtered = json_aggregate_item(db.CallStack, filter_kwargs, id) if call_stack == None: raise cherrypy.HTTPError(404) call_stack[1] = str(call_stack[1]) #unicode throws off template when casting dict as js obj call_stack[2] = int(call_stack[2]) #convert long to int mytemplate = Template(filename=os.path.join(self.templates_dir,'aggregatecallstack.html'), lookup=self.template_lookup) return mytemplate.render(call_stack=call_stack, kwargs=filter_kwargs) else: mytemplate = Template(filename=os.path.join(self.templates_dir,'aggregatecallstacks.html'), lookup=self.template_lookup) return mytemplate.render(kwargs=filter_kwargs) @cherrypy.expose def sqlstatements(self, id=None, **kwargs): table_kwargs, filter_kwargs = parse_kwargs(kwargs) if 'id' in filter_kwargs: filter_kwargs.pop('id') for k in filter_kwargs: filter_kwargs[k] = str(filter_kwargs[k]) if id: statement, total, filtered = json_aggregate_item(db.SQLStatement, filter_kwargs, id) if statement == None: raise cherrypy.HTTPError(404) statement[1] = str(statement[1]) #unicode throws off template when casting dict as js obj statement[2] = int(statement[2]) #convert long to int mytemplate = Template(filename=os.path.join(self.templates_dir,'aggregatesql.html'), lookup=self.template_lookup) return mytemplate.render(statement=statement, kwargs=filter_kwargs) else: mytemplate = Template(filename=os.path.join(self.templates_dir,'aggregatesqls.html'), lookup=self.template_lookup) return mytemplate.render(kwargs=filter_kwargs) @cherrypy.expose def fileaccesses(self, id=None, **kwargs): table_kwargs, filter_kwargs = parse_kwargs(kwargs) if 'id' in filter_kwargs: filter_kwargs.pop('id') for k in filter_kwargs: filter_kwargs[k] = str(filter_kwargs[k]) if id: file_access, total, filtered = json_aggregate_item(db.FileAccess, filter_kwargs, id) if file_access == None: raise cherrypy.HTTPError(404) file_access[1] = str(file_access[1]) #unicode throws off template when casting dict as js obj file_access[2] = int(file_access[2]) #convert long to int mytemplate = Template(filename=os.path.join(self.templates_dir,'aggregatefileaccess.html'), lookup=self.template_lookup) return mytemplate.render(file_access=file_access, kwargs=filter_kwargs) else: mytemplate = Template(filename=os.path.join(self.templates_dir,'aggregatefileaccesses.html'), lookup=self.template_lookup) return mytemplate.render(kwargs=filter_kwargs)
.. moduleauthor:: `Marie FETIVEAU <github.com/mfe>`_ """ __version__ = "0.0" import cherrypy from mako.lookup import TemplateLookup import os import ntpath import traceback from plotThatLut import plot_that_lut from utils import matplotlib_helper as mplh mplh.WEB_MODE = True CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) MY_LOOKUP = TemplateLookup(directories=[os.path.join(CURRENT_DIR, 'html')]) class Application(object): """CherryPy Application """ @cherrypy.expose def index(self): """ColorPipe-tools index page """ mytemplate = MY_LOOKUP.get_template("index.html") return mytemplate.render() @cherrypy.expose
from otest.rp.setup import as_arg_setup from otest.rp.tool import WebTester try: from requests.packages import urllib3 except ImportError: pass else: urllib3.disable_warnings() __author__ = 'roland' ROOT = './' LOOKUP = TemplateLookup(directories=[ROOT + 'htdocs'], module_directory=ROOT + 'modules', input_encoding='utf-8', output_encoding='utf-8') logger = logging.getLogger("") def setup_logging(logfile_name, log_level=logging.DEBUG): hdlr = logging.FileHandler(logfile_name) base_formatter = logging.Formatter( "%(asctime)s %(name)s:%(levelname)s %(message)s") hdlr.setFormatter(base_formatter) logger.addHandler(hdlr) logger.setLevel(log_level)
import re import httplib import json import cherrypy import gdg import bcrypt from urlparse import urljoin, urlparse from mako.template import Template from mako.lookup import TemplateLookup from gdg.data import * # Content is relative to the base directory, not the module directory. current_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) virtual_dir = '' templates = TemplateLookup(directories=['html'], strict_undefined=True) application = None class ImageModel(object): def __init__(self, **entries): self.__dict__.update(entries) def get_relative_path(base, path): if path == None: return None if not path == "": path = os.path.relpath(path, current_dir) return urljoin(base, path.replace('\\', '/'))
def __init__(self, template_directory): self.lookup = TemplateLookup(directories=[template_directory], strict_undefined=True)
def resource_path(relative): """Makes relative to package""" return pkg_resources.resource_filename(__name__, relative) #TODO: fix support here for template lookups, internal, user provided #template_cache_dir = config.template_cache_dir template_cache_dir = "cache" #disable cache for cleaner folder structure #TODO: Also try for Cisco build here lookup = TemplateLookup(directories=[resource_path("")], #module_directory= template_cache_dir, cache_type='memory', cache_enabled=True, ) try: import autonetkit_cisco # test if can import, if not present will fail and not add to template path cisco_templates = pkg_resources.resource_filename("autonetkit_cisco", "") lookup.directories.append(cisco_templates) except ImportError: pass # Cisco ANK not present #TODO: make a render class, that caches traversed folders for speed #TODO: Add support for both src template and src folder (eg for quagga, servers) def render_node(node, folder_cache): log.debug("Rendering %s" % node)
def load_environment(global_conf, app_conf): """ Configure the Pylons environment via the ``pylons.config`` object """ # Pylons paths root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) paths = dict(root=root, controllers=os.path.join(root, 'controllers'), static_files=os.path.join(root, 'public'), templates=[ app_conf.get('custom_templates', os.path.join(root, 'templates')), os.path.join(root, 'templates') ]) # Initialize config with the basic options config.init_app(global_conf, app_conf, package='linotp', paths=paths) config['linotp.root'] = root config['routes.map'] = make_map() config['pylons.app_globals'] = app_globals.Globals() config['pylons.h'] = linotp.lib.helpers ## add per token a location for the mako template lookup ## @note: the location is defined in the .ini file by ## the entry [linotpTokenModules] directories = paths['templates'] ## add a template path for every token modules = get_token_module_list() for module in modules: mpath = os.path.dirname(module.__file__) directories.append(mpath) ## add a template path for every resolver modules = get_resolver_module_list() for module in modules: mpath = os.path.dirname(module.__file__) directories.append(mpath) unique_directories = _uniqify_list(directories) log.debug("[load_environment] Template directories: %r" % unique_directories) config['pylons.app_globals'].mako_lookup = TemplateLookup( directories=unique_directories, error_handler=handle_mako_error, module_directory=os.path.join(app_conf['cache_dir'], 'templates'), input_encoding='utf-8', default_filters=['escape'], imports=['from webhelpers.html import escape']) # Setup the SQLAlchemy database engine # If we load the linotp.model here, the pylons.config is loaded with # the entries from the config file. if it is loaded at the top of the file, #the pylons.config does not contain the config file, yet. from linotp.model import init_model engine = engine_from_config(config, 'sqlalchemy.') init_model(engine) # CONFIGURATION OPTIONS HERE (note: all config options will override # any Pylons config options) from linotp.lib.audit.base import getAudit audit = getAudit() config['audit'] = audit ## setup Security provider definition try: log.debug('[load_environment] loading token list definition') g = config['pylons.app_globals'] g.security_provider.load_config(config) except Exception as e: log.error("Failed to load security provider definition: %r" % e) raise e ## load the list of tokenclasses try: log.debug('[load_environment] loading token list definition') (tcl, tpl) = get_token_class_list() config['tokenclasses'] = tcl g.setTokenclasses(tcl) config['tokenprefixes'] = tpl g.setTokenprefixes(tpl) except Exception as e: log.error("Failed to load token class list: %r" % e) raise e ## load the list of resolvers try: log.debug('[load_environment] loading resolver list definition') (rclass, rname) = get_resolver_class_list() ## make this globaly avaliable g.setResolverClasses(rclass) g.setResolverTypes(rname) except Exception as exx: log.error("Faild to load the list of resolvers: %r" % exx) raise exx ## get the help url url = config.get("linotpHelp.url", None) if url is None: version = pkg_resources.get_distribution("linotp").version # First try to get the help for this specific version url = "http://linotp.org/doc/%s/index.html" % version config['help_url'] = url log.debug("[load_environment] done") return
def _write_httpd_conf(pubs, default_pubs, runtime_dir, log_dir, template_dir, cache_dir, cache_size, host, port, sroot, fragment=False, allow_refresh=False, ssl_cert_file="", ssl_key_file="", ssl_cert_chain_file=""): """Writes the webserver configuration for the depot. pubs repository and publisher information, a list in the form [(publisher_prefix, repo_dir, repo_prefix, writable_root), ... ] default_pubs default publishers, per repository, a list in the form [(default_publisher_prefix, repo_dir, repo_prefix) ... ] runtime_dir where we write httpd.conf files log_dir where Apache should write its log files template_dir where we find our Mako templates cache_dir where Apache should write its cache and wsgi search idx cache_size how large our cache can grow host our hostname, needed to set ServerName properly port the port on which Apache should listen sroot the prefix into the server namespace, ignored if fragment==False fragment True if we should only write a file to drop into conf.d/ (i.e. a partial server configuration) allow_refresh True if we allow the 'refresh' or 'refresh-indexes' admin/0 operations The URI namespace we create on the web server looks like this: <sroot>/<repo_prefix>/<publisher>/<file, catalog etc.>/<version>/ <sroot>/<repo_prefix>/<file, catalog etc.>/<version>/ 'sroot' is only used when the Apache server is serving other content and we want to separate pkg(5) resources from the other resources provided. 'repo_prefix' exists so that we can disambiguate between multiple repositories that provide the same publisher. 'ssl_cert_file' the location of the server certificate file. 'ssl_key_file' the location of the server key file. 'ssl_cert_chain_file' the location of the certificate chain file if the the server certificate is not signed by the top level CA. """ try: # check our hostname socket.getaddrinfo(host, None) # Apache needs IPv6 addresses wrapped in square brackets if ":" in host: host = "[{0}]".format(host) # check our directories dirs = [runtime_dir] if not fragment: dirs.append(log_dir) if cache_dir: dirs.append(cache_dir) for dir in dirs + [template_dir]: if os.path.exists(dir) and not os.path.isdir(dir): raise DepotException(_("{0} is not a directory").format(dir)) for dir in dirs: misc.makedirs(dir) # check our port if not fragment: try: num = int(port) if num <= 0 or num >= 65535: raise DepotException(_("invalid port: {0}").format(port)) except ValueError: raise DepotException(_("invalid port: {0}").format(port)) # check our cache size try: num = int(cache_size) if num < 0: raise DepotException( _("invalid cache size: " "{0}").format(num)) except ValueError: raise DepotException( _("invalid cache size: {0}").format(cache_size)) httpd_conf_template_path = os.path.join(template_dir, DEPOT_HTTP_TEMPLATE) fragment_conf_template_path = os.path.join(template_dir, DEPOT_FRAGMENT_TEMPLATE) # we're disabling unicode here because we want Mako to # passthrough any filesystem path names, whatever the # original encoding. conf_lookup = TemplateLookup(directories=[template_dir]) disable_unicode = True if six.PY2 else False if fragment: conf_template = Template(filename=fragment_conf_template_path, disable_unicode=disable_unicode, lookup=conf_lookup) conf_path = os.path.join(runtime_dir, DEPOT_FRAGMENT_FILENAME) else: conf_template = Template(filename=httpd_conf_template_path, disable_unicode=disable_unicode, lookup=conf_lookup) conf_path = os.path.join(runtime_dir, DEPOT_HTTP_FILENAME) conf_text = conf_template.render( pubs=pubs, default_pubs=default_pubs, log_dir=log_dir, cache_dir=cache_dir, cache_size=cache_size, runtime_dir=runtime_dir, template_dir=template_dir, ipv6_addr="::1", host=host, port=port, sroot=sroot, allow_refresh=allow_refresh, ssl_cert_file=ssl_cert_file, ssl_key_file=ssl_key_file, ssl_cert_chain_file=ssl_cert_chain_file) with open(conf_path, "w") as conf_file: conf_file.write(conf_text) except (socket.gaierror, UnicodeError) as err: # socket.getaddrinfo raise UnicodeDecodeError in Python 3 # for some input, such as '.' raise DepotException( _("Unable to write Apache configuration: {host}: " "{err}").format(**locals())) except (OSError, IOError, EnvironmentError, apx.ApiException) as err: traceback.print_exc() raise DepotException( _("Unable to write depot_httpd.conf: {0}").format(err))
def get_lookup(): mylookup = TemplateLookup(directories=['./templates'], output_encoding='utf-8', encoding_errors='replace') mytemplate = mylookup.get_template("base.html") return None
import json from django.conf import settings from django.http import HttpResponse from django.template.context import Context from mako.lookup import TemplateLookup from common.log import logger """ mylookup 也可以单独使用: @Example: mako_temp = mylookup.get_template(template_name) mako_temp.render(**data) """ mylookup = TemplateLookup( directories=settings.MAKO_TEMPLATE_DIR, module_directory=settings.MAKO_TEMPLATE_MODULE_DIR, output_encoding='utf-8', input_encoding='utf-8', encoding_errors='replace', collection_size=500, ) def render_mako(template_name, dictionary={}, context_instance=None): """ render the mako template and return the HttpResponse @param template_name: 模板名字 @param dictionary: context字典 @param context_instance: 初始化context,如果要使用 TEMPLATE_CONTEXT_PROCESSORS,则要使用RequestContext(request) @note: 因为返回是HttpResponse,所以这个方法也适合给ajax用(dataType不是json的ajax) @Example: render_mako('mako_temp.html',{'form':form})
# add current folder to system path sys.path.insert(0, os.getcwd()) imports = [] try: # you can use mysite module to hold global variables import mysite imports.append('import mysite') except ImportError: print('Please create module mysite.py to keep global vars!') lookup = TemplateLookup(directories=['.'], input_encoding='utf-8', output_encoding='utf-8', encoding_errors='replace', imports=imports) class StoppableHTTPServer(HTTPServer): """HTTPServer that can be interrupted by pressing Ctrl-C""" def run(self): (hostaddr, port) = self.socket.getsockname() print("Serving HTTP on {} port {} ...".format(hostaddr, port)) h = '127.0.0.1' if hostaddr == '0.0.0.0' else hostaddr print("Open http://{}:{} in your browser!".format(h, port)) try: self.serve_forever() except KeyboardInterrupt: # Clean-up server (close socket, etc.)
from osf.models.base import BaseModel, ObjectIDMixin from osf.models.external import ExternalAccount from osf.models.node import AbstractNode from osf.models.user import OSFUser from osf.utils.datetime_aware_jsonfield import DateTimeAwareJSONField from website import settings from addons.base import logger, serializer from website.oauth.signals import oauth_complete lookup = TemplateLookup( directories=[settings.TEMPLATES_PATH], default_filters=[ 'unicode', # default filter; must set explicitly when overriding # FIXME: Temporary workaround for data stored in wrong format in DB. Unescape it before it # gets re-escaped by Markupsafe. See [#OSF-4432] 'temp_ampersand_fixer', 'h', ], imports=[ # FIXME: Temporary workaround for data stored in wrong format in DB. Unescape it before it # gets re-escaped by Markupsafe. See [#OSF-4432] 'from website.util.sanitize import temp_ampersand_fixer', ]) class BaseAddonSettings(ObjectIDMixin, BaseModel): deleted = models.BooleanField(default=False) class Meta: abstract = True @property
def build_templatelookup(args): # Set up the location the templates will be served out of. return TemplateLookup(directories=[args.filedir], module_directory=args.cachedir, collection_size=100)
#Kaithem Automation is free software: you can redistribute it and/or modify #it under the terms of the GNU General Public License as published by #the Free Software Foundation, version 3. #Kaithem Automation is distributed in the hope that it will be useful, #but WITHOUT ANY WARRANTY; without even the implied warranty of #MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the #GNU General Public License for more details. #You should have received a copy of the GNU General Public License #along with Kaithem Automation. If not, see <http://www.gnu.org/licenses/>. from mako.template import Template from mako.lookup import TemplateLookup import auth import cherrypy _Lookup = TemplateLookup(directories=['pages']) get_template = _Lookup.get_template #Redirect user to an error message if he does not have the proper permission. def require(permission): if (not 'auth' in cherrypy.request.cookie ) or cherrypy.request.cookie['auth'].value not in auth.Tokens: raise cherrypy.HTTPRedirect("/login/") if not auth.checkTokenPermission(cherrypy.request.cookie['auth'].value, permission): raise cherrypy.HTTPRedirect("/errors/permissionerror")
def main(): parser = argparse.ArgumentParser() parser.add_argument('--out', help='Path to the output directory', required=True) parser.add_argument('--staging', help='Path to the staging directory', required=True) parser.add_argument('--zircon-user-build', help='Path to the Zircon "user" build directory', required=True) parser.add_argument('--zircon-tool-build', help='Path to the Zircon "tools" build directory', required=True) parser.add_argument('--debug', help='Whether to print out debug information', action='store_true') args = parser.parse_args() out_dir = os.path.abspath(args.out) shutil.rmtree(os.path.join(out_dir, 'config'), True) shutil.rmtree(os.path.join(out_dir, 'fidl'), True) shutil.rmtree(os.path.join(out_dir, 'lib'), True) shutil.rmtree(os.path.join(out_dir, 'sysroot'), True) shutil.rmtree(os.path.join(out_dir, 'tool'), True) debug = args.debug # Generate package descriptions through Zircon's build. zircon_dir = os.path.abspath(args.staging) shutil.rmtree(zircon_dir, True) if debug: print('Building Zircon in: %s' % zircon_dir) make_args = [ 'make', 'packages', 'BUILDDIR=%s' % zircon_dir, ] env = {} env['PATH'] = os.environ['PATH'] if not debug: env['QUIET'] = '1' subprocess.check_call(make_args, cwd=ZIRCON_ROOT, env=env) # Parse package definitions. packages = [] with open(os.path.join(zircon_dir, 'export', 'manifest'), 'r') as manifest: package_files = map(lambda line: line.strip(), manifest.readlines()) for file in package_files: with open(os.path.join(zircon_dir, 'export', file), 'r') as pkg_file: packages.append(parse_package(pkg_file.readlines())) if debug: print('Found %s packages:' % len(packages)) names = sorted(map(lambda p: p['package']['name'], packages)) for name in names: print(' - %s' % name) # Generate some GN glue for each package. context = GenerationContext( out_dir, ZIRCON_ROOT, os.path.abspath(args.zircon_user_build), os.path.abspath(args.zircon_tool_build), TemplateLookup(directories=[SCRIPT_DIR]), ) for package in packages: name = package['package']['name'] type = package['package']['type'] arch = package['package']['arch'] if name == 'c': generate_sysroot(package, context) print('Generated sysroot') continue if name in SYSROOT_PACKAGES: print('Ignoring sysroot part: %s' % name) continue if type == 'tool': generate_host_tool(package, context) elif type == 'lib': if arch == 'src': type = 'source' generate_source_library(package, context) else: type = 'prebuilt' generate_compiled_library(package, context) elif type == 'fidl': generate_fidl_library(package, context) else: print('(%s) Unsupported package type: %s/%s, skipping' % (name, type, arch)) continue if debug: print('Processed %s (%s)' % (name, type)) board_path = os.path.join(zircon_dir, 'export', 'all-boards.list') with open(board_path, 'r') as board_file: package = parse_package(board_file.readlines()) generate_board_list(package, context)
def create_lookup(self): """Create a template lookup.""" self.lookup = TemplateLookup( directories=self.directories, module_directory=self.cache_dir, output_encoding='utf-8')
def __init__(self): self.lookup = TemplateLookup(directories=['.'], output_encoding='utf-8', encoding_errors='replace')
def __init__(self, *a, **kwargs): from mako.lookup import TemplateLookup self._lookup = TemplateLookup(*a, **kwargs)
class SpiderFootWebUi: lookup = TemplateLookup(directories=['']) defaultConfig = dict() config = dict() token = None docroot = '' def __init__(self, config): """Initialize web server Args: config: TBD """ if not isinstance(config, dict): raise TypeError("config is %s; expected dict()" % type(config)) if not config: raise ValueError("config is empty") self.defaultConfig = deepcopy(config) dbh = SpiderFootDb(self.defaultConfig) # 'config' supplied will be the defaults, let's supplement them # now with any configuration which may have previously been # saved. sf = SpiderFoot(self.defaultConfig) self.config = sf.configUnserialize(dbh.configGet(), self.defaultConfig) if self.config['__webaddr'] == "0.0.0.0": # nosec addr = "<IP of this host>" else: addr = self.config['__webaddr'] self.docroot = self.config['__docroot'].rstrip('/') cherrypy.config.update({ 'error_page.404': self.error_page_404, 'request.error_response': self.error_page }) secure_headers = SecureHeaders( server="server", cache=False, csp= "default-src 'self' ; script-src 'self' 'unsafe-inline' blob: ; style-src 'self' 'unsafe-inline' ; img-src 'self' data:" ) cherrypy.config.update({ "tools.response_headers.on": True, "tools.response_headers.headers": secure_headers.cherrypy() }) if (cherrypy.server.ssl_certificate is None or cherrypy.server.ssl_private_key is None): url = "http://%s:%s%s" % (addr, self.config['__webport'], self.docroot) else: url = "https://%s:%s%s" % (addr, self.config['__webport'], self.docroot) print("") print("") print("*************************************************************") print(" Use SpiderFoot by starting your web browser of choice and ") print(" browse to %s" % url) print("*************************************************************") print("") print("") def error_page(self): """Error page""" cherrypy.response.status = 500 if self.config['_debug']: cherrypy.response.body = _cperror.get_error_page( status=500, traceback=_cperror.format_exc()) else: cherrypy.response.body = '<html><body>Error</body></html>' def error_page_404(self, status, message, traceback, version): """Error page 404 Args: status: TBD message: TBD traceback: TBD version: TBD """ templ = Template(filename='dyn/error.tmpl', lookup=self.lookup) return templ.render(message='Not Found', docroot=self.docroot, status=status) def cleanUserInput(self, inputList): """Sanitize user input, poorly. Args: inputList (list): TBD Returns: list: sanitized input Raises: TypeError: inputList type was invalid """ if not isinstance(inputList, list): raise TypeError("inputList is %s; expected list()" % type(inputList)) ret = list() for item in inputList: c = html.escape(item, True) c = c.replace('\'', '"') # We don't actually want & translated to & c = c.replace("&", "&").replace(""", "\"") ret.append(c) return ret def searchBase(self, id=None, eventType=None, value=None): """Search Args: id: TBD eventType: TBD value: TBD Returns: list: search results """ retdata = [] regex = "" if [id, eventType, value].count('') == 3 or [id, eventType, value ].count(None) == 3: return retdata if value.startswith("/") and value.endswith("/"): regex = value[1:len(value) - 1] value = "" value = value.replace('*', '%') if value in [None, ""] and regex in [None, ""]: value = "%" regex = "" dbh = SpiderFootDb(self.config) criteria = { 'scan_id': None if id == '' else id, 'type': None if eventType == '' else eventType, 'value': None if value == '' else value, 'regex': None if regex == '' else regex } try: data = dbh.search(criteria) except Exception: return retdata for row in data: lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0])) escapeddata = html.escape(row[1]) escapedsrc = html.escape(row[2]) retdata.append([ lastseen, escapeddata, escapedsrc, row[3], row[5], row[6], row[7], row[8], row[10], row[11], row[4], row[13], row[14] ]) return retdata # # USER INTERFACE PAGES # def scaneventresultexport(self, id, type, dialect="excel"): """Get scan event result data in CSV format Args: id (str): scan ID type (str): TBD dialect (str): TBD Returns: string: results in CSV format """ dbh = SpiderFootDb(self.config) data = dbh.scanResultEvent(id, type) fileobj = StringIO() parser = csv.writer(fileobj, dialect=dialect) parser.writerow(["Updated", "Type", "Module", "Source", "F/P", "Data"]) for row in data: if row[4] == "ROOT": continue lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0])) datafield = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "") parser.writerow([ lastseen, str(row[4]), str(row[3]), str(row[2]), row[13], datafield ]) cherrypy.response.headers[ 'Content-Disposition'] = "attachment; filename=SpiderFoot.csv" cherrypy.response.headers['Content-Type'] = "application/csv" cherrypy.response.headers['Pragma'] = "no-cache" return fileobj.getvalue().encode("utf-8") scaneventresultexport.exposed = True def scaneventresultexportmulti(self, ids, dialect="excel"): """Get scan event result data in CSV format for multiple scans Args: ids (str): comma separated list of scan IDs dialect (str): TBD Returns: string: results in CSV format """ dbh = SpiderFootDb(self.config) scaninfo = dict() data = list() for id in ids.split(','): scaninfo[id] = dbh.scanInstanceGet(id) data = data + dbh.scanResultEvent(id) fileobj = StringIO() parser = csv.writer(fileobj, dialect=dialect) parser.writerow([ "Scan Name", "Updated", "Type", "Module", "Source", "F/P", "Data" ]) for row in data: if row[4] == "ROOT": continue lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0])) datafield = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "") parser.writerow([ scaninfo[row[12]][0], lastseen, str(row[4]), str(row[3]), str(row[2]), row[13], datafield ]) cherrypy.response.headers[ 'Content-Disposition'] = "attachment; filename=SpiderFoot.csv" cherrypy.response.headers['Content-Type'] = "application/csv" cherrypy.response.headers['Pragma'] = "no-cache" return fileobj.getvalue().encode("Utf-8") scaneventresultexportmulti.exposed = True # Get search result data in CSV format def scansearchresultexport(self, id, eventType=None, value=None, dialect="excel"): data = self.searchBase(id, eventType, value) fileobj = StringIO() parser = csv.writer(fileobj, dialect=dialect) parser.writerow(["Updated", "Type", "Module", "Source", "F/P", "Data"]) if not data: return None for row in data: if row[10] == "ROOT": continue datafield = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "") parser.writerow([ row[0], str(row[10]), str(row[3]), str(row[2]), row[11], datafield ]) cherrypy.response.headers[ 'Content-Disposition'] = "attachment; filename=SpiderFoot.csv" cherrypy.response.headers['Content-Type'] = "application/csv" cherrypy.response.headers['Pragma'] = "no-cache" return fileobj.getvalue().encode("Utf-8") scansearchresultexport.exposed = True # Export results from multiple scans in JSON format def scanexportjsonmulti(self, ids): dbh = SpiderFootDb(self.config) scaninfo = list() scan_name = "" for id in ids.split(','): scan = dbh.scanInstanceGet(id) if scan is None: continue scan_name = scan[0] for row in dbh.scanResultEvent(id): lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0])) event_data = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "") source_data = str(row[2]) source_module = str(row[3]) event_type = row[4] false_positive = row[13] if event_type == "ROOT": continue scaninfo.append({ "data": event_data, "event_type": event_type, "module": source_module, "source_data": source_data, "false_positive": false_positive, "last_seen": lastseen, "scan_name": scan_name, "scan_target": scan[1] }) if len(ids.split(',')) > 1 or scan_name == "": fname = "SpiderFoot.json" else: fname = scan_name + "-SpiderFoot.json" cherrypy.response.headers[ 'Content-Disposition'] = "attachment; filename=" + fname cherrypy.response.headers[ 'Content-Type'] = "application/json; charset=utf-8" cherrypy.response.headers['Pragma'] = "no-cache" return json.dumps(scaninfo).encode("utf-8") scanexportjsonmulti.exposed = True # Export entities from scan results for visualising def scanviz(self, id, gexf="0"): dbh = SpiderFootDb(self.config) sf = SpiderFoot(self.config) data = dbh.scanResultEvent(id, filterFp=True) scan = dbh.scanInstanceGet(id) root = scan[1] if gexf != "0": cherrypy.response.headers[ 'Content-Disposition'] = "attachment; filename=SpiderFoot.gexf" cherrypy.response.headers['Content-Type'] = "application/gexf" cherrypy.response.headers['Pragma'] = "no-cache" return sf.buildGraphGexf([root], "SpiderFoot Export", data) else: return sf.buildGraphJson([root], data) scanviz.exposed = True # Export entities results from multiple scans in GEXF format def scanvizmulti(self, ids, gexf="1"): dbh = SpiderFootDb(self.config) sf = SpiderFoot(self.config) data = list() roots = list() for id in ids.split(','): data = data + dbh.scanResultEvent(id, filterFp=True) roots.append(dbh.scanInstanceGet(id)[1]) if gexf != "0": cherrypy.response.headers[ 'Content-Disposition'] = "attachment; filename=SpiderFoot.gexf" cherrypy.response.headers['Content-Type'] = "application/gexf" cherrypy.response.headers['Pragma'] = "no-cache" return sf.buildGraphGexf(roots, "SpiderFoot Export", data) else: # Not implemented yet return None scanvizmulti.exposed = True def scanopts(self, id): """Configuration used for a scan Args: id: scan ID """ ret = dict() dbh = SpiderFootDb(self.config) ret['config'] = dbh.scanConfigGet(id) ret['configdesc'] = dict() for key in list(ret['config'].keys()): if ':' not in key: ret['configdesc'][key] = self.config['__globaloptdescs__'][key] else: [modName, modOpt] = key.split(':') if modName not in list(self.config['__modules__'].keys()): continue if modOpt not in list(self.config['__modules__'][modName] ['optdescs'].keys()): continue ret['configdesc'][key] = self.config['__modules__'][modName][ 'optdescs'][modOpt] meta = dbh.scanInstanceGet(id) if not meta: return json.dumps([]) if meta[3] != 0: started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(meta[3])) else: started = "Not yet" if meta[4] != 0: finished = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(meta[4])) else: finished = "Not yet" ret['meta'] = [meta[0], meta[1], meta[2], started, finished, meta[5]] return json.dumps(ret) scanopts.exposed = True def rerunscan(self, id): """Rerun a scan Args: id (str): scan ID """ # Snapshot the current configuration to be used by the scan cfg = deepcopy(self.config) modlist = list() sf = SpiderFoot(cfg) dbh = SpiderFootDb(cfg) info = dbh.scanInstanceGet(id) if not info: return self.error("Invalid scan ID.") scanconfig = dbh.scanConfigGet(id) scanname = info[0] scantarget = info[1] targetType = None if len(scanconfig) == 0: return self.error("Something went wrong internally.") modlist = scanconfig['_modulesenabled'].split(',') if "sfp__stor_stdout" in modlist: modlist.remove("sfp__stor_stdout") targetType = sf.targetType(scantarget) if targetType is None: # It must then be a name, as a re-run scan should always have a clean # target. Put quotes around the target value and try to determine the # target type again. targetType = sf.targetType(f'"{scantarget}"') if targetType != "HUMAN_NAME": scantarget = scantarget.lower() # Start running a new scan scanId = sf.genScanInstanceGUID() try: p = mp.Process(target=SpiderFootScanner, args=(scanname, scanId, scantarget, targetType, modlist, cfg)) p.daemon = True p.start() except BaseException as e: print("[-] Scan [%s] failed: %s" % (scanId, e)) return self.error("Scan [%s] failed: %s" % (scanId, e)) # Wait until the scan has initialized while dbh.scanInstanceGet(scanId) is None: print("[info] Waiting for the scan to initialize...") time.sleep(1) raise cherrypy.HTTPRedirect(f"scaninfo?id={scanId}", status=302) rerunscan.exposed = True def rerunscanmulti(self, ids): """Rerun scans Args: ids (str): comma separated list of scan IDs """ # Snapshot the current configuration to be used by the scan cfg = deepcopy(self.config) modlist = list() sf = SpiderFoot(cfg) dbh = SpiderFootDb(cfg) for id in ids.split(","): info = dbh.scanInstanceGet(id) scanconfig = dbh.scanConfigGet(id) scanname = info[0] scantarget = info[1] targetType = None if len(scanconfig) == 0: return self.error("Something went wrong internally.") modlist = scanconfig['_modulesenabled'].split(',') if "sfp__stor_stdout" in modlist: modlist.remove("sfp__stor_stdout") targetType = sf.targetType(scantarget) if targetType is None: # Should never be triggered for a re-run scan.. return self.error( "Invalid target type. Could not recognize it as a human name, IP address, IP subnet, ASN, domain name or host name." ) # Start running a new scan scanId = sf.genScanInstanceGUID() try: p = mp.Process(target=SpiderFootScanner, args=(scanname, scanId, scantarget, targetType, modlist, cfg)) p.daemon = True p.start() except BaseException as e: print("[-] Scan [%s] failed: %s" % (scanId, e)) return self.error("Scan [%s] failed: %s" % (scanId, e)) # Wait until the scan has initialized while dbh.scanInstanceGet(scanId) is None: print("[info] Waiting for the scan to initialize...") time.sleep(1) templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup) return templ.render(rerunscans=True, docroot=self.docroot, pageid="SCANLIST") rerunscanmulti.exposed = True def newscan(self): """ Configure a new scan """ dbh = SpiderFootDb(self.config) types = dbh.eventTypes() templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup) return templ.render(pageid='NEWSCAN', types=types, docroot=self.docroot, modules=self.config['__modules__'], scanname="", selectedmods="", scantarget="") newscan.exposed = True def clonescan(self, id): """ Clone an existing scan (pre-selected options in the newscan page) Args: id (str): scan ID to clone """ sf = SpiderFoot(self.config) dbh = SpiderFootDb(self.config) types = dbh.eventTypes() info = dbh.scanInstanceGet(id) if not info: return self.error("Invalid scan ID.") scanconfig = dbh.scanConfigGet(id) scanname = info[0] scantarget = info[1] targetType = None if scanname == "" or scantarget == "" or len(scanconfig) == 0: return self.error("Something went wrong internally.") targetType = sf.targetType(scantarget) if targetType is None: # It must be a name, so wrap quotes around it scantarget = """ + scantarget + """ modlist = scanconfig['_modulesenabled'].split(',') templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup) return templ.render(pageid='NEWSCAN', types=types, docroot=self.docroot, modules=self.config['__modules__'], selectedmods=modlist, scanname=str(scanname), scantarget=str(scantarget)) clonescan.exposed = True def index(self): """ Main page listing scans available """ templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup) return templ.render(pageid='SCANLIST', docroot=self.docroot) index.exposed = True def scaninfo(self, id): """Information about a selected scan Args: id (str): scan id """ dbh = SpiderFootDb(self.config) res = dbh.scanInstanceGet(id) if res is None: return self.error("Scan ID not found.") templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup, input_encoding='utf-8') return templ.render(id=id, name=html.escape(res[0]), status=res[5], docroot=self.docroot, pageid="SCANLIST") scaninfo.exposed = True def opts(self, updated=None): """Settings Args: updated: TBD """ templ = Template(filename='dyn/opts.tmpl', lookup=self.lookup) self.token = random.SystemRandom().randint(0, 99999999) return templ.render(opts=self.config, pageid='SETTINGS', token=self.token, updated=updated, docroot=self.docroot) opts.exposed = True def optsexport(self, pattern): """Export configuration Args: pattern: TBD """ sf = SpiderFoot(self.config) conf = sf.configSerialize(self.config) content = "" for opt in sorted(conf): if ":_" in opt or opt.startswith("_"): continue if pattern: if pattern in opt: content += "%s=%s\n" % (opt, conf[opt]) else: content += "%s=%s\n" % (opt, conf[opt]) cherrypy.response.headers[ 'Content-Disposition'] = 'attachment; filename="SpiderFoot.cfg"' cherrypy.response.headers['Content-Type'] = "text/plain" return content optsexport.exposed = True def optsraw(self): """Settings""" ret = dict() self.token = random.SystemRandom().randint(0, 99999999) for opt in self.config: if not opt.startswith('__'): ret["global." + opt] = self.config[opt] continue if opt == '__modules__': for mod in sorted(self.config['__modules__'].keys()): for mo in sorted( self.config['__modules__'][mod]['opts'].keys()): if mo.startswith("_"): continue ret["module." + mod + "." + mo] = self.config['__modules__'][mod]['opts'][mo] return json.dumps(['SUCCESS', {'token': self.token, 'data': ret}]) optsraw.exposed = True def error(self, message): """Generic error, but not exposed as not called directly Args: message (str): error message """ templ = Template(filename='dyn/error.tmpl', lookup=self.lookup) return templ.render(message=message, docroot=self.docroot) def scandelete(self, id, confirm=None, raw=False): """Delete a scan Args: id (str): scan ID confirm: TBD raw: TBD """ dbh = SpiderFootDb(self.config) res = dbh.scanInstanceGet(id) if res is None: if raw: return json.dumps(["ERROR", "Scan ID not found."]) return self.error("Scan ID not found.") if confirm: dbh.scanInstanceDelete(id) if raw: return json.dumps(["SUCCESS", ""]) raise cherrypy.HTTPRedirect("/") templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup) return templ.render(id=id, name=str(res[0]), names=list(), ids=list(), pageid="SCANLIST", docroot=self.docroot) scandelete.exposed = True def scandeletemulti(self, ids, confirm=None): """Delete a scan Args: ids (str): comma separated list of scan IDs confirm: TBD """ dbh = SpiderFootDb(self.config) names = list() for id in ids.split(','): res = dbh.scanInstanceGet(id) names.append(str(res[0])) if res is None: return self.error("Scan ID not found (" + id + ").") if res[5] in ["RUNNING", "STARTING", "STARTED"]: return self.error("You cannot delete running scans.") if confirm: for id in ids.split(','): dbh.scanInstanceDelete(id) raise cherrypy.HTTPRedirect("/") templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup) return templ.render(id=None, name=None, ids=ids.split(','), names=names, pageid="SCANLIST", docroot=self.docroot) scandeletemulti.exposed = True def savesettings(self, allopts, token, configFile=None): """Save settings, also used to completely reset them to default Args: allopts: TBD token: TBD configFile: TBD """ if str(token) != str(self.token): return self.error("Invalid token (" + str(self.token) + ").") if configFile: # configFile seems to get set even if a file isn't uploaded if configFile.file: contents = configFile.file.read() if type(contents) == bytes: contents = contents.decode("utf-8") try: tmp = dict() for line in contents.split("\n"): if "=" not in line: continue opt_array = line.strip().split("=") if len(opt_array) == 1: opt_array[1] = "" tmp[opt_array[0]] = '='.join(opt_array[1:]) allopts = json.dumps(tmp) except BaseException as e: return self.error( "Failed to parse input file. Was it generated from SpiderFoot? (%s)" % e) # Reset config to default if allopts == "RESET": if self.reset_settings(): raise cherrypy.HTTPRedirect("/opts?updated=1") else: return self.error("Failed to reset settings") # Save settings try: dbh = SpiderFootDb(self.config) useropts = json.loads(allopts) cleanopts = dict() for opt in list(useropts.keys()): cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0] currentopts = deepcopy(self.config) # Make a new config where the user options override # the current system config. sf = SpiderFoot(self.config) self.config = sf.configUnserialize(cleanopts, currentopts) dbh.configSet(sf.configSerialize(self.config)) except Exception as e: return self.error( "Processing one or more of your inputs failed: %s" % e) raise cherrypy.HTTPRedirect("/opts?updated=1") savesettings.exposed = True def savesettingsraw(self, allopts, token): """Save settings, also used to completely reset them to default Args: allopts: TBD token: TBD """ if str(token) != str(self.token): return json.dumps( ["ERROR", "Invalid token (" + str(self.token) + ")."]) # Reset config to default if allopts == "RESET": if self.reset_settings(): return json.dumps(["SUCCESS", ""]) else: return json.dumps(["ERROR", "Failed to reset settings"]) # Save settings try: dbh = SpiderFootDb(self.config) useropts = json.loads(allopts) cleanopts = dict() for opt in list(useropts.keys()): cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0] currentopts = deepcopy(self.config) # Make a new config where the user options override # the current system config. sf = SpiderFoot(self.config) self.config = sf.configUnserialize(cleanopts, currentopts) dbh.configSet(sf.configSerialize(self.config)) except Exception as e: return json.dumps([ "ERROR", "Processing one or more of your inputs failed: %s" % e ]) return json.dumps(["SUCCESS", ""]) savesettingsraw.exposed = True def reset_settings(self): """Reset settings to default""" try: dbh = SpiderFootDb(self.config) dbh.configClear() # Clear it in the DB self.config = deepcopy(self.defaultConfig) # Clear in memory except Exception: return False return True def resultsetfp(self, id, resultids, fp): """Set a bunch of results (hashes) as false positive Args: id (str): scan ID resultids (str): comma separated list of result IDs fp (str): 0 or 1 """ dbh = SpiderFootDb(self.config) if fp not in ["0", "1"]: return json.dumps( ["ERROR", "No FP flag set or not set correctly."]) ids = json.loads(resultids) if not ids: return json.dumps(["ERROR", "No IDs supplied."]) # Cannot set FPs if a scan is not completed status = dbh.scanInstanceGet(id) if not status: return self.error("Invalid scan ID: %s" % id) if status[5] not in ["ABORTED", "FINISHED", "ERROR-FAILED"]: return json.dumps([ "WARNING", "Scan must be in a finished state when setting False Positives." ]) # Make sure the user doesn't set something as non-FP when the # parent is set as an FP. if fp == "0": data = dbh.scanElementSourcesDirect(id, ids) for row in data: if str(row[14]) == "1": return json.dumps([ "WARNING", "Cannot unset element %s as False Positive if a parent element is still False Positive." % id ]) # Set all the children as FPs too.. it's only logical afterall, right? childs = dbh.scanElementChildrenAll(id, ids) allIds = ids + childs ret = dbh.scanResultsUpdateFP(id, allIds, fp) if ret: return json.dumps(["SUCCESS", ""]) return json.dumps(["ERROR", "Exception encountered."]) resultsetfp.exposed = True def eventtypes(self): """For the CLI to fetch a list of event types. Returns: str: list of event types """ dbh = SpiderFootDb(self.config) types = dbh.eventTypes() ret = list() for r in types: ret.append([r[1], r[0]]) ret = sorted(ret, key=itemgetter(0)) return json.dumps(ret) eventtypes.exposed = True def modules(self): """For the CLI to fetch a list of modules. Returns: str: list of modules """ modinfo = list(self.config['__modules__'].keys()) modinfo.sort() ret = list() for m in modinfo: if "__" in m: continue ret.append({ 'name': m, 'descr': self.config['__modules__'][m]['descr'] }) return json.dumps(ret) modules.exposed = True def ping(self): """For the CLI to test connectivity to this server.""" return json.dumps(["SUCCESS", self.config['__version__']]) ping.exposed = True def query(self, query): """For the CLI to run queries against the database.""" data = None dbh = SpiderFootDb(self.config) cherrypy.response.headers[ 'Content-Type'] = "application/json; charset=utf-8" if not query: return json.dumps(["ERROR", "Invalid query."]) if not query.lower().startswith("select"): return json.dumps([ "ERROR", "Non-SELECTs are unpredictable and not recommended." ]) try: ret = dbh.dbh.execute(query) data = ret.fetchall() columnNames = [c[0] for c in dbh.dbh.description] data = [dict(zip(columnNames, row)) for row in data] except BaseException as e: return json.dumps(["ERROR", str(e)]) return json.dumps(data) query.exposed = True def startscan(self, scanname, scantarget, modulelist, typelist, usecase, cli=None): """Initiate a scan Args: scanname (str): scan name scantarget (str): scan target modulelist (str): TBD typelist (str): TBD usecase (str): TBD cli: TBD """ # Swap the globalscantable for the database handler dbh = SpiderFootDb(self.config) # Snapshot the current configuration to be used by the scan cfg = deepcopy(self.config) modlist = list() sf = SpiderFoot(cfg) targetType = None [scanname, scantarget] = self.cleanUserInput([scanname, scantarget]) if scanname == "" or scantarget == "": if cli: return json.dumps([ "ERROR", "Incorrect usage: scan name or target was not specified." ]) return self.error( "Invalid request: scan name or target was not specified.") if typelist == "" and modulelist == "" and usecase == "": if cli: return json.dumps([ "ERROR", "Incorrect usage: no modules specified for scan." ]) return self.error( "Invalid request: no modules specified for scan.") # User selected modules if modulelist != "": modlist = modulelist.replace('module_', '').split(',') # User selected types if len(modlist) == 0 and typelist != "": typesx = typelist.replace('type_', '').split(',') # 1. Find all modules that produce the requested types modlist = sf.modulesProducing(typesx) newmods = deepcopy(modlist) newmodcpy = deepcopy(newmods) # 2. For each type those modules consume, get modules producing while len(newmodcpy) > 0: for etype in sf.eventsToModules(newmodcpy): xmods = sf.modulesProducing([etype]) for mod in xmods: if mod not in modlist: modlist.append(mod) newmods.append(mod) newmodcpy = deepcopy(newmods) newmods = list() # User selected a use case if len(modlist) == 0 and usecase != "": for mod in self.config['__modules__']: if usecase == 'all' or usecase in self.config['__modules__'][ mod]['cats']: modlist.append(mod) # Add our mandatory storage module.. if "sfp__stor_db" not in modlist: modlist.append("sfp__stor_db") modlist.sort() targetType = sf.targetType(scantarget) if targetType is None: if cli: return json.dumps(["ERROR", "Unrecognised target type."]) return self.error( "Invalid target type. Could not recognize it as a human name, IP address, IP subnet, ASN, domain name or host name." ) # Delete the stdout module in case it crept in if "sfp__stor_stdout" in modlist: modlist.remove("sfp__stor_stdout") # Start running a new scan if targetType in ["HUMAN_NAME", "USERNAME"]: scantarget = scantarget.replace("\"", "") else: scantarget = scantarget.lower() # Start running a new scan scanId = sf.genScanInstanceGUID() try: p = mp.Process(target=SpiderFootScanner, args=(scanname, scanId, scantarget, targetType, modlist, cfg)) p.daemon = True p.start() except BaseException as e: print("[-] Scan [%s] failed: %s" % (scanId, e)) return self.error("Scan [%s] failed: %s" % (scanId, e)) # Wait until the scan has initialized # Check the database for the scan status results while dbh.scanInstanceGet(scanId) is None: print("[info] Waiting for the scan to initialize...") time.sleep(1) if cli: return json.dumps(["SUCCESS", scanId]) raise cherrypy.HTTPRedirect(f"/scaninfo?id={scanId}") startscan.exposed = True def stopscanmulti(self, ids): """Stop a scan Args: ids (str): comma separated list of scan IDs Note: Unnecessary for now given that only one simultaneous scan is permitted """ dbh = SpiderFootDb(self.config) error = list() for id in ids.split(","): scaninfo = dbh.scanInstanceGet(id) if not scaninfo: return self.error("Invalid scan ID: %s" % id) scanname = str(scaninfo[0]) scanstatus = scaninfo[5] if scanstatus == "FINISHED": error.append( "Scan '%s' is in a finished state. <a href='/scandelete?id=%s&confirm=1'>Maybe you want to delete it instead?</a>" % (scanname, id)) continue if scanstatus == "ABORTED": error.append("Scan '" + scanname + "' is already aborted.") continue dbh.scanInstanceSet(id, status="ABORT-REQUESTED") raise cherrypy.HTTPRedirect("/") stopscanmulti.exposed = True def stopscan(self, id, cli=None): """Stop a scan. Args: id (str): scan ID cli: TBD """ dbh = SpiderFootDb(self.config) scaninfo = dbh.scanInstanceGet(id) if not scaninfo: if cli: return json.dumps(["ERROR", "Invalid scan ID."]) return self.error("Invalid scan ID.") scanstatus = scaninfo[5] if scanstatus == "ABORTED": if cli: return json.dumps(["ERROR", "Scan already aborted."]) return self.error("The scan is already aborted.") if not scanstatus == "RUNNING": if cli: return json.dumps( ["ERROR", "Scan in an invalid state for stopping."]) return self.error( "The running scan is currently in the state '%s', please try again later or restart SpiderFoot." % scanstatus) dbh.scanInstanceSet(id, status="ABORT-REQUESTED") if cli: return json.dumps(["SUCCESS", ""]) raise cherrypy.HTTPRedirect("/") stopscan.exposed = True # # DATA PROVIDERS # def scanlog(self, id, limit=None, rowId=None, reverse=None): """Scan log data Args: id: TBD limit: TBD rowId: TBD reverse: TBD """ dbh = SpiderFootDb(self.config) retdata = [] try: data = dbh.scanLogs(id, limit, rowId, reverse) except Exception: return json.dumps(retdata) for row in data: generated = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0] / 1000)) retdata.append( [generated, row[1], row[2], html.escape(row[3]), row[4]]) return json.dumps(retdata) scanlog.exposed = True def scanerrors(self, id, limit=None): """Scan error data Args: id (str): scan ID limit: TBD """ dbh = SpiderFootDb(self.config) retdata = [] try: data = dbh.scanErrors(id, limit) except Exception: return json.dumps(retdata) for row in data: generated = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0] / 1000)) retdata.append([generated, row[1], html.escape(str(row[2]))]) return json.dumps(retdata) scanerrors.exposed = True def scanlist(self): """Produce a list of scans""" dbh = SpiderFootDb(self.config) data = dbh.scanInstanceList() retdata = [] for row in data: created = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[3])) if row[4] == 0: started = "Not yet" else: started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[4])) if row[5] == 0: finished = "Not yet" else: finished = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[5])) retdata.append([ row[0], row[1], row[2], created, started, finished, row[6], row[7] ]) return json.dumps(retdata) scanlist.exposed = True def scanstatus(self, id): """Basic information about a scan Args: id (str): scan ID """ dbh = SpiderFootDb(self.config) data = dbh.scanInstanceGet(id) if not data: return json.dumps([]) created = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[2])) started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[3])) ended = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[4])) retdata = [data[0], data[1], created, started, ended, data[5]] return json.dumps(retdata) scanstatus.exposed = True def scansummary(self, id, by): """Summary of scan results Args: id (str): scan ID by: TBD """ retdata = [] dbh = SpiderFootDb(self.config) try: scandata = dbh.scanResultSummary(id, by) except Exception: return json.dumps(retdata) try: statusdata = dbh.scanInstanceGet(id) except Exception: return json.dumps(retdata) for row in scandata: if row[0] == "ROOT": continue lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[2])) retdata.append( [row[0], row[1], lastseen, row[3], row[4], statusdata[5]]) return json.dumps(retdata) scansummary.exposed = True def scaneventresults(self, id, eventType, filterfp=False): """Event results for a scan Args: id (str): scan ID eventType (str): filter by event type filterfp: TBD """ retdata = [] dbh = SpiderFootDb(self.config) try: data = dbh.scanResultEvent(id, eventType, filterfp) except Exception: return json.dumps(retdata) for row in data: lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0])) escapeddata = html.escape(row[1]) escapedsrc = html.escape(row[2]) retdata.append([ lastseen, escapeddata, escapedsrc, row[3], row[5], row[6], row[7], row[8], row[13], row[14], row[4] ]) return json.dumps(retdata, ensure_ascii=False) scaneventresults.exposed = True def scaneventresultsunique(self, id, eventType, filterfp=False): """Unique event results for a scan Args: id (str): scan ID eventType (str): filter by event type filterfp: TBD """ dbh = SpiderFootDb(self.config) retdata = [] try: data = dbh.scanResultEventUnique(id, eventType, filterfp) except Exception: return json.dumps(retdata) for row in data: escaped = html.escape(row[0]) retdata.append([escaped, row[1], row[2]]) return json.dumps(retdata, ensure_ascii=False) scaneventresultsunique.exposed = True def search(self, id=None, eventType=None, value=None): """Search Args: id: TBD eventType (str): filter by event type value: TBD """ try: data = self.searchBase(id, eventType, value) except Exception: return json.dumps([]) return json.dumps(data, ensure_ascii=False) search.exposed = True def scanhistory(self, id): """Historical data for a scan. Args: id (str): scan ID """ dbh = SpiderFootDb(self.config) try: data = dbh.scanResultHistory(id) except Exception: return json.dumps([]) return json.dumps(data, ensure_ascii=False) scanhistory.exposed = True def scanelementtypediscovery(self, id, eventType): """scan element type discovery Args: id: TBD eventType (str): filter by event type """ sf = SpiderFoot(self.config) dbh = SpiderFootDb(self.config) pc = dict() datamap = dict() # Get the events we will be tracing back from leafSet = dbh.scanResultEvent(id, eventType) [datamap, pc] = dbh.scanElementSourcesAll(id, leafSet) # Delete the ROOT key as it adds no value from a viz perspective del pc['ROOT'] retdata = dict() retdata['tree'] = sf.dataParentChildToTree(pc) retdata['data'] = datamap return json.dumps(retdata, ensure_ascii=False) scanelementtypediscovery.exposed = True
from flask import Flask, request app = Flask(__name__) from mako.template import Template as MakoTemplates from mako.lookup import TemplateLookup from jinja2 import Environment as Jinja2Environment import tornado.template import random import string import time mylookup = TemplateLookup(directories=['/tpl']) Jinja2Env = Jinja2Environment(line_statement_prefix='#') def shutdown_server(): func = request.environ.get('werkzeug.server.shutdown') if func is None: raise RuntimeError('Not running with the Werkzeug Server') func() def randomword(length=8): return ''.join(random.choice(string.lowercase) for i in range(length)) @app.route("/reflect/<engine>") def reflect(engine): template = request.values.get('tpl') if not template:
def __init__(self, short_name, full_name, owners, categories, added_default=None, added_mandatory=None, node_settings_model=None, user_settings_model=None, include_js=None, include_css=None, widget_help=None, views=None, configs=None, models=None, has_hgrid_files=False, get_hgrid_data=None, max_file_size=None, high_max_file_size=None, accept_extensions=True, node_settings_template=None, user_settings_template=None, **kwargs): self.models = models self.settings_models = {} if node_settings_model: node_settings_model.config = self self.settings_models['node'] = node_settings_model if user_settings_model: user_settings_model.config = self self.settings_models['user'] = user_settings_model self.short_name = short_name self.full_name = full_name self.owners = owners self.categories = categories self.added_default = added_default or [] self.added_mandatory = added_mandatory or [] if set(self.added_mandatory).difference(self.added_default): raise ValueError('All mandatory targets must also be defaults.') self.include_js = self._include_to_static(include_js or {}) self.include_css = self._include_to_static(include_css or {}) self.widget_help = widget_help self.views = views or [] self.configs = configs or [] self.has_hgrid_files = has_hgrid_files # WARNING: get_hgrid_data can return None if the addon is added but has no credentials. self.get_hgrid_data = get_hgrid_data # if has_hgrid_files and not get_hgrid_data rubeus.make_dummy() self.max_file_size = max_file_size self.high_max_file_size = high_max_file_size self.accept_extensions = accept_extensions # Build template lookup template_path = os.path.join('website', 'addons', short_name, 'templates') if os.path.exists(template_path): self.template_lookup = TemplateLookup(directories=[ template_path, settings.TEMPLATES_PATH, ]) else: self.template_lookup = None # Provide the path the the user_settings template self.user_settings_template = user_settings_template addon_user_settings_path = os.path.join( template_path, '{}_user_settings.mako'.format(self.short_name)) if user_settings_template: # If USER_SETTINGS_TEMPLATE is defined, use that path. self.user_settings_template = user_settings_template elif os.path.exists(addon_user_settings_path): # An implicit template exists self.user_settings_template = os.path.join( os.path.pardir, 'addons', self.short_name, 'templates', '{}_user_settings.mako'.format(self.short_name), ) else: # Use the default template (for OAuth addons) self.user_settings_template = os.path.join( 'project', 'addon', 'user_settings_default.mako', ) self.node_settings_template = node_settings_template
def __init__(path): self._lookup = TemplateLookup(directories=[path], module_directory='mako_modules')
help="The ID of the entities descriptor") parser.add_argument('-k', dest='keyfile', help="A file with a key to sign the metadata with") parser.add_argument('-n', dest='name') parser.add_argument('-s', dest='sign', action='store_true', help="sign the metadata") parser.add_argument('-m', dest='mako_root', default="./") parser.add_argument(dest="config") args = parser.parse_args() _rot = args.mako_root LOOKUP = TemplateLookup(directories=[_rot + 'templates', _rot + 'htdocs'], module_directory=_rot + 'modules', input_encoding='utf-8', output_encoding='utf-8') PORT = 8088 AUTHN_BROKER = AuthnBroker() AUTHN_BROKER.add(authn_context_class_ref(PASSWORD), username_password_authn, 10, "http://%s" % socket.gethostname()) AUTHN_BROKER.add(authn_context_class_ref(UNSPECIFIED), "", 0, "http://%s" % socket.gethostname()) IDP = server.Server(args.config, cache=Cache()) IDP.ticket = {} SRV = make_server('', PORT, application)
def main(name): template_lookup = TemplateLookup(directories=[name]) template = template_lookup.get_template('welcome.html') return lambda ctx: template.render(**ctx)