def newscan(self): dbh = SpiderFootDb(self.config) types = dbh.eventTypes() templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup) return templ.render(pageid='NEWSCAN', types=types, docroot=self.docroot, modules=self.config['__modules__'], scanname="", selectedmods="", scantarget="")
def clonescan(self, id): sf = SpiderFoot(self.config) dbh = SpiderFootDb(self.config) types = dbh.eventTypes() info = dbh.scanInstanceGet(id) scanconfig = dbh.scanConfigGet(id) scanname = info[0] scantarget = info[1] targetType = None if scanname == "" or scantarget == "" or len(scanconfig) == 0: return self.error("Something went wrong internally.") targetType = sf.targetType(scantarget) if targetType == None: # It must be a name, so wrap quotes around it scantarget = """ + scantarget + """ modlist = scanconfig['_modulesenabled'].split(',') templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup) return templ.render(pageid='NEWSCAN', types=types, docroot=self.docroot, modules=self.config['__modules__'], selectedmods=modlist, scanname=str(scanname), scantarget=str(scantarget))
def clonescan(self, id): sf = SpiderFoot(self.config) dbh = SpiderFootDb(self.config) types = dbh.eventTypes() info = dbh.scanInstanceGet(id) scanconfig = dbh.scanConfigGet(id) scanname = info[0] scantarget = info[1] targetType = None if scanname == "" or scantarget == "" or len(scanconfig) == 0: return self.error("Something went wrong internally.") targetType = sf.targetType(scantarget) if targetType == None: # It must be a name, so wrap quotes around it scantarget = """ + scantarget + """ modlist = scanconfig['_modulesenabled'].split(',') templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup) return templ.render(pageid='NEWSCAN', types=types, docroot=self.docroot, modules=self.config['__modules__'], selectedmods=modlist, scanname=unicode(scanname, 'utf-8', errors='replace'), scantarget=unicode(scantarget, 'utf-8', errors='replace'))
def test_event_types_should_return_a_list(self): """ Test eventTypes(self) """ sfdb = SpiderFootDb(self.default_options, False) event_types = sfdb.eventTypes() self.assertIsInstance(event_types, list)
def startscan(self, scanname, scantarget, modulelist, typelist): modopts = dict() # Not used yet as module options are set globally modlist = list() sf = SpiderFoot(self.config) dbh = SpiderFootDb(self.config) types = dbh.eventTypes() [scanname, scantarget] = self.cleanUserInput([scanname, scantarget]) if scanname == "" or scantarget == "": return self.error("Form incomplete.") if typelist == "" and modulelist == "": return self.error("Form incomplete.") if modulelist != "": modlist = modulelist.replace('module_', '').split(',') else: typesx = typelist.replace('type_', '').split(',') # 1. Find all modules that produce the requested types modlist = sf.modulesProducing(typesx) newmods = deepcopy(modlist) newmodcpy = deepcopy(newmods) # 2. For each type those modules consume, get modules producing while len(newmodcpy) > 0: for etype in sf.eventsToModules(newmodcpy): xmods = sf.modulesProducing([etype]) for mod in xmods: if mod not in modlist: modlist.append(mod) newmods.append(mod) newmodcpy = deepcopy(newmods) newmods = list() # Add our mandatory storage module.. if "sfp__stor_db" not in modlist: modlist.append("sfp__stor_db") modlist.sort() # For now we don't permit multiple simultaneous scans for thread in threading.enumerate(): if thread.name.startswith("SF_"): templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup) return templ.render(modules=self.config['__modules__'], alreadyRunning=True, runningScan=thread.name[3:], types=types, pageid="NEWSCAN") # Start running a new scan self.scanner = SpiderFootScanner(scanname, scantarget.lower(), modlist, self.config, modopts) t = threading.Thread(name="SF_" + scanname, target=self.scanner.startScan) t.start() templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup) return templ.render(id=self.scanner.myId, name=scanname, status=self.scanner.status, pageid="SCANLIST")
def eventtypes(self): dbh = SpiderFootDb(self.config) types = dbh.eventTypes() ret = list() for r in types: ret.append([r[1], r[0]]) ret = sorted(ret, key=itemgetter(0)) return json.dumps(ret)
def eventtypes(self): """For the CLI to fetch a list of event types. Returns: str: list of event types """ dbh = SpiderFootDb(self.config) types = dbh.eventTypes() ret = list() for r in types: ret.append([r[1], r[0]]) ret = sorted(ret, key=itemgetter(0)) return json.dumps(ret)
def clonescan(self, id): dbh = SpiderFootDb(self.config) types = dbh.eventTypes() info = dbh.scanInstanceGet(id) scanconfig = dbh.scanConfigGet(id) scanname = info[0] scantarget = info[1] targetType = None if scanname == "" or scantarget == "" or len(scanconfig) == 0: return self.error("Something went wrong internally.") modlist = scanconfig['_modulesenabled'].split(',') templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup) return templ.render(pageid='NEWSCAN', types=types, docroot=self.docroot, modules=self.config['__modules__'], selectedmods=modlist, scanname=scanname, scantarget=scantarget)
def startscan(self, scanname, scantarget, modulelist, typelist): global globalScanStatus # Snapshot the current configuration to be used by the scan cfg = deepcopy(self.config) modopts = dict() # Not used yet as module options are set globally modlist = list() sf = SpiderFoot(cfg) dbh = SpiderFootDb(cfg) types = dbh.eventTypes() targetType = None [scanname, scantarget] = self.cleanUserInput([scanname, scantarget]) if scanname == "" or scantarget == "": return self.error("Form incomplete.") if typelist == "" and modulelist == "": return self.error("Form incomplete.") if modulelist != "": modlist = modulelist.replace('module_', '').split(',') else: typesx = typelist.replace('type_', '').split(',') # 1. Find all modules that produce the requested types modlist = sf.modulesProducing(typesx) newmods = deepcopy(modlist) newmodcpy = deepcopy(newmods) # 2. For each type those modules consume, get modules producing while len(newmodcpy) > 0: for etype in sf.eventsToModules(newmodcpy): xmods = sf.modulesProducing([etype]) for mod in xmods: if mod not in modlist: modlist.append(mod) newmods.append(mod) newmodcpy = deepcopy(newmods) newmods = list() # Add our mandatory storage module.. if "sfp__stor_db" not in modlist: modlist.append("sfp__stor_db") modlist.sort() targetType = sf.targetType(scantarget) if targetType is None: return self.error("Invalid target type. Could not recognize it as " + \ "an IP address, IP subnet, domain name or host name.") # Start running a new scan scanId = sf.genScanInstanceGUID(scanname) t = SpiderFootScanner(scanname, scantarget.lower(), targetType, scanId, modlist, cfg, modopts) t.start() # Wait until the scan has initialized while globalScanStatus.getStatus(scanId) is None: print "[info] Waiting for the scan to initialize..." time.sleep(1) templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup) return templ.render(id=scanId, name=scanname, docroot=self.docroot, status=globalScanStatus.getStatus(scanId), pageid="SCANLIST")
def newscan(self): dbh = SpiderFootDb(self.config) types = dbh.eventTypes() templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup) return templ.render(pageid='NEWSCAN', types=types, modules=self.config['__modules__'])
def startscan(self, scanname, scantarget, modulelist, typelist): global globalScanStatus # Snapshot the current configuration to be used by the scan cfg = deepcopy(self.config) modopts = dict() # Not used yet as module options are set globally modlist = list() sf = SpiderFoot(cfg) dbh = SpiderFootDb(cfg) types = dbh.eventTypes() targetType = None [scanname, scantarget] = self.cleanUserInput([scanname, scantarget]) if scanname == "" or scantarget == "": return self.error("Form incomplete.") if typelist == "" and modulelist == "": return self.error("Form incomplete.") if modulelist != "": modlist = modulelist.replace('module_', '').split(',') else: typesx = typelist.replace('type_', '').split(',') # 1. Find all modules that produce the requested types modlist = sf.modulesProducing(typesx) newmods = deepcopy(modlist) newmodcpy = deepcopy(newmods) # 2. For each type those modules consume, get modules producing while len(newmodcpy) > 0: for etype in sf.eventsToModules(newmodcpy): xmods = sf.modulesProducing([etype]) for mod in xmods: if mod not in modlist: modlist.append(mod) newmods.append(mod) newmodcpy = deepcopy(newmods) newmods = list() # Add our mandatory storage module.. if "sfp__stor_db" not in modlist: modlist.append("sfp__stor_db") modlist.sort() regexToType = { "^\d+\.\d+\.\d+\.\d+$": "IP_ADDRESS", "^\d+\.\d+\.\d+\.\d+/\d+$": "NETBLOCK_OWNER", "^.[a-zA-Z\-0-9\.]+$": "INTERNET_NAME" } # Parse the target and set the targetType for rx in regexToType.keys(): if re.match(rx, scantarget, re.IGNORECASE): targetType = regexToType[rx] break if targetType == None: return self.error("Invalid target type. Could not recognize it as " + \ "an IP address, IP subnet, domain name or host name.") # Start running a new scan scanId = sf.genScanInstanceGUID(scanname) t = SpiderFootScanner(scanname, scantarget.lower(), targetType, scanId, modlist, cfg, modopts) t.start() # Wait until the scan has initialized while globalScanStatus.getStatus(scanId) == None: print "[info] Waiting for the scan to initialize..." time.sleep(1) templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup) return templ.render(id=scanId, name=scanname, status=globalScanStatus.getStatus(scanId), pageid="SCANLIST")
def eventtypes(self): dbh = SpiderFootDb(self.config) types = dbh.eventTypes() return json.dumps(types)
sf = SpiderFoot(sfConfig) dbh = SpiderFootDb(sfConfig, init=True) if not args.l: if args.modules: print("Modules available:") for m in sorted(sfModules.keys()): if "__" in m: continue print(('{0:25} {1}'.format(m, sfModules[m]['descr']))) sys.exit(0) if args.types: print("Types available:") typedata = dbh.eventTypes() types = dict() for r in typedata: types[r[1]] = r[0] for t in sorted(types.keys()): print(('{0:45} {1}'.format(t, types[t]))) sys.exit(0) if not args.s: print( "You must specify a target when running in scan mode. Try --help for guidance." ) sys.exit(-1) if args.x and not args.t:
def startscan(self, scanname, scantarget, modulelist, typelist): modopts = dict() # Not used yet as module options are set globally modlist = list() sf = SpiderFoot(self.config) dbh = SpiderFootDb(self.config) types = dbh.eventTypes() [scanname, scantarget] = self.cleanUserInput([scanname, scantarget]) if scanname == "" or scantarget == "": return self.error("Form incomplete.") if typelist == "" and modulelist == "": return self.error("Form incomplete.") if modulelist != "": modlist = modulelist.replace('module_', '').split(',') else: typesx = typelist.replace('type_', '').split(',') # 1. Find all modules that produce the requested types modlist = sf.modulesProducing(typesx) newmods = deepcopy(modlist) newmodcpy = deepcopy(newmods) # 2. For each type those modules consume, get modules producing while len(newmodcpy) > 0: for etype in sf.eventsToModules(newmodcpy): xmods = sf.modulesProducing([etype]) for mod in xmods: if mod not in modlist: modlist.append(mod) newmods.append(mod) newmodcpy = deepcopy(newmods) newmods = list() # Add our mandatory storage module.. if "sfp__stor_db" not in modlist: modlist.append("sfp__stor_db") modlist.sort() # For now we don't permit multiple simultaneous scans for thread in threading.enumerate(): if thread.name.startswith("SF_"): templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup) return templ.render(modules=self.config['__modules__'], alreadyRunning=True, runningScan=thread.name[3:], types=types, pageid="NEWSCAN") # Start running a new scan self.scanner = SpiderFootScanner(scanname, scantarget.lower(), modlist, self.config, modopts) t = threading.Thread(name="SF_" + scanname, target=self.scanner.startScan) t.start() # Spin cycles waiting for the scan ID to be set while self.scanner.myId == None: time.sleep(1) continue templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup) return templ.render(id=self.scanner.myId, name=scanname, status=self.scanner.status, pageid="SCANLIST")
def startscan(self, scanname, scantarget, modulelist, typelist, usecase): global globalScanStatus # Snapshot the current configuration to be used by the scan cfg = deepcopy(self.config) modopts = dict() # Not used yet as module options are set globally modlist = list() sf = SpiderFoot(cfg) dbh = SpiderFootDb(cfg) types = dbh.eventTypes() targetType = None [scanname, scantarget] = self.cleanUserInput([scanname, scantarget]) if scanname == "" or scantarget == "": return self.error("Form incomplete.") if typelist == "" and modulelist == "" and usecase == "": return self.error("Form incomplete.") # User selected modules if modulelist != "": modlist = modulelist.replace('module_', '').split(',') # User selected types if len(modlist) == 0 and typelist != "": typesx = typelist.replace('type_', '').split(',') # 1. Find all modules that produce the requested types modlist = sf.modulesProducing(typesx) newmods = deepcopy(modlist) newmodcpy = deepcopy(newmods) # 2. For each type those modules consume, get modules producing while len(newmodcpy) > 0: for etype in sf.eventsToModules(newmodcpy): xmods = sf.modulesProducing([etype]) for mod in xmods: if mod not in modlist: modlist.append(mod) newmods.append(mod) newmodcpy = deepcopy(newmods) newmods = list() # User selected a use case if len(modlist) == 0 and usecase != "": for mod in self.config['__modules__']: if usecase == 'all' or usecase in self.config['__modules__'][ mod]['cats']: modlist.append(mod) # Add our mandatory storage module.. if "sfp__stor_db" not in modlist: modlist.append("sfp__stor_db") modlist.sort() targetType = sf.targetType(scantarget) if targetType is None: return self.error("Invalid target type. Could not recognize it as " + \ "an IP address, IP subnet, domain name or host name.") # Start running a new scan scanId = sf.genScanInstanceGUID(scanname) t = SpiderFootScanner(scanname, scantarget.lower(), targetType, scanId, modlist, cfg, modopts) t.start() # Wait until the scan has initialized while globalScanStatus.getStatus(scanId) is None: print "[info] Waiting for the scan to initialize..." time.sleep(1) templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup) return templ.render(id=scanId, name=scanname, docroot=self.docroot, status=globalScanStatus.getStatus(scanId), pageid="SCANLIST")
sf = SpiderFoot(sfConfig) dbh = SpiderFootDb(sfConfig, init=True) if args: if args.modules: print "Modules available:" for m in sorted(sfModules.keys()): if "__" in m: continue print '{0:25} {1}'.format(m, sfModules[m]['descr']) sys.exit(0) if args.types: print "Types available:" typedata = dbh.eventTypes() types = dict() for r in typedata: types[r[1]] = r[0] for t in sorted(types.keys()): print '{0:45} {1}'.format(t, types[t]) sys.exit(0) if not args.s: print "You must specify a target when running in scan mode. Try sf.py --help for guidance." sys.exit(-1) if args.x and not args.t: print "-x can only be used with -t. Use --help for guidance." sys.exit(-1)