def test_target_type(self): """ Test targetType(target) """ target_type = SpiderFootHelpers.targetTypeFromString("0.0.0.0") self.assertEqual('IP_ADDRESS', target_type) target_type = SpiderFootHelpers.targetTypeFromString( "*****@*****.**") self.assertEqual('EMAILADDR', target_type) target_type = SpiderFootHelpers.targetTypeFromString("0.0.0.0/0") self.assertEqual('NETBLOCK_OWNER', target_type) target_type = SpiderFootHelpers.targetTypeFromString("+1234567890") self.assertEqual('PHONE_NUMBER', target_type) target_type = SpiderFootHelpers.targetTypeFromString('"Human Name"') self.assertEqual('HUMAN_NAME', target_type) target_type = SpiderFootHelpers.targetTypeFromString('"abc123"') self.assertEqual('USERNAME', target_type) target_type = SpiderFootHelpers.targetTypeFromString("1234567890") self.assertEqual('BGP_AS_OWNER', target_type) target_type = SpiderFootHelpers.targetTypeFromString("::1") self.assertEqual('IPV6_ADDRESS', target_type) target_type = SpiderFootHelpers.targetTypeFromString("spiderfoot.net") self.assertEqual('INTERNET_NAME', target_type) target_type = SpiderFootHelpers.targetTypeFromString( "1HesYJSP1QqcyPEjnQ9vzBL1wujruNGe7R") self.assertEqual('BITCOIN_ADDRESS', target_type)
def rerunscan(self, id): """Rerun a scan Args: id (str): scan ID Returns: None Raises: HTTPRedirect: redirect to info page for new scan """ # Snapshot the current configuration to be used by the scan cfg = deepcopy(self.config) modlist = list() dbh = SpiderFootDb(cfg) info = dbh.scanInstanceGet(id) if not info: return self.error("Invalid scan ID.") scanname = info[0] scantarget = info[1] scanconfig = dbh.scanConfigGet(id) if not scanconfig: return self.error(f"Error loading config from scan: {id}") modlist = scanconfig['_modulesenabled'].split(',') if "sfp__stor_stdout" in modlist: modlist.remove("sfp__stor_stdout") targetType = SpiderFootHelpers.targetTypeFromString(scantarget) if not targetType: # It must then be a name, as a re-run scan should always have a clean # target. Put quotes around the target value and try to determine the # target type again. targetType = SpiderFootHelpers.targetTypeFromString(f'"{scantarget}"') if targetType not in ["HUMAN_NAME", "BITCOIN_ADDRESS"]: scantarget = scantarget.lower() # Start running a new scan scanId = SpiderFootHelpers.genScanInstanceId() try: p = mp.Process(target=SpiderFootScanner, args=(scanname, scanId, scantarget, targetType, modlist, cfg)) p.daemon = True p.start() except Exception as e: self.log.error(f"[-] Scan [{scanId}] failed: {e}") return self.error(f"[-] Scan [{scanId}] failed: {e}") # Wait until the scan has initialized while dbh.scanInstanceGet(scanId) is None: self.log.info("Waiting for the scan to initialize...") time.sleep(1) raise cherrypy.HTTPRedirect(f"{self.docroot}/scaninfo?id={scanId}", status=302)
def test_target_type_invalid_seed_should_return_none(self): """ Test targetType(target) """ target_type = SpiderFootHelpers.targetTypeFromString(None) self.assertEqual(None, target_type) target_type = SpiderFootHelpers.targetTypeFromString("") self.assertEqual(None, target_type) target_type = SpiderFootHelpers.targetTypeFromString('""') self.assertEqual(None, target_type)
def rerunscanmulti(self, ids): """Rerun scans Args: ids (str): comma separated list of scan IDs Returns: None """ # Snapshot the current configuration to be used by the scan cfg = deepcopy(self.config) modlist = list() dbh = SpiderFootDb(cfg) for id in ids.split(","): info = dbh.scanInstanceGet(id) if not info: return self.error("Invalid scan ID.") scanconfig = dbh.scanConfigGet(id) scanname = info[0] scantarget = info[1] targetType = None if len(scanconfig) == 0: return self.error("Something went wrong internally.") modlist = scanconfig['_modulesenabled'].split(',') if "sfp__stor_stdout" in modlist: modlist.remove("sfp__stor_stdout") targetType = SpiderFootHelpers.targetTypeFromString(scantarget) if targetType is None: # Should never be triggered for a re-run scan.. return self.error("Invalid target type. Could not recognize it as a target SpiderFoot supports.") # Start running a new scan scanId = SpiderFootHelpers.genScanInstanceId() try: p = mp.Process(target=SpiderFootScanner, args=(scanname, scanId, scantarget, targetType, modlist, cfg)) p.daemon = True p.start() except Exception as e: self.log.error(f"[-] Scan [{scanId}] failed: {e}") return self.error(f"[-] Scan [{scanId}] failed: {e}") # Wait until the scan has initialized while dbh.scanInstanceGet(scanId) is None: self.log.info("Waiting for the scan to initialize...") time.sleep(1) templ = Template(filename='spiderfoot/templates/scanlist.tmpl', lookup=self.lookup) return templ.render(rerunscans=True, docroot=self.docroot, pageid="SCANLIST", version=__version__)
def clonescan(self, id): """ Clone an existing scan (pre-selected options in the newscan page) Args: id (str): scan ID to clone Returns: None """ dbh = SpiderFootDb(self.config) types = dbh.eventTypes() info = dbh.scanInstanceGet(id) if not info: return self.error("Invalid scan ID.") scanconfig = dbh.scanConfigGet(id) scanname = info[0] scantarget = info[1] targetType = None if scanname == "" or scantarget == "" or len(scanconfig) == 0: return self.error("Something went wrong internally.") targetType = SpiderFootHelpers.targetTypeFromString(scantarget) if targetType is None: # It must be a name, so wrap quotes around it scantarget = """ + scantarget + """ modlist = scanconfig['_modulesenabled'].split(',') templ = Template(filename='spiderfoot/templates/newscan.tmpl', lookup=self.lookup) return templ.render(pageid='NEWSCAN', types=types, docroot=self.docroot, modules=self.config['__modules__'], selectedmods=modlist, scanname=str(scanname), scantarget=str(scantarget), version=__version__)
def start_scan(sfConfig: dict, sfModules: dict, args, loggingQueue) -> None: """Start scan Args: sfConfig (dict): SpiderFoot config options sfModules (dict): modules args (argparse.Namespace): command line args loggingQueue (Queue): main SpiderFoot logging queue """ log = logging.getLogger(f"spiderfoot.{__name__}") global dbh global scanId dbh = SpiderFootDb(sfConfig, init=True) sf = SpiderFoot(sfConfig) if not args.s: log.error( "You must specify a target when running in scan mode. Try --help for guidance." ) sys.exit(-1) if args.x and not args.t: log.error("-x can only be used with -t. Use --help for guidance.") sys.exit(-1) if args.x and args.m: log.error( "-x can only be used with -t and not with -m. Use --help for guidance." ) sys.exit(-1) if args.r and (args.o and args.o not in ["tab", "csv"]): log.error("-r can only be used when your output format is tab or csv.") sys.exit(-1) if args.H and (args.o and args.o not in ["tab", "csv"]): log.error("-H can only be used when your output format is tab or csv.") sys.exit(-1) if args.D and args.o != "csv": log.error("-D can only be used when using the csv output format.") sys.exit(-1) target = args.s # Usernames and names - quoted on the commandline - won't have quotes, # so add them. if " " in target: target = f"\"{target}\"" if "." not in target and not target.startswith("+") and '"' not in target: target = f"\"{target}\"" targetType = SpiderFootHelpers.targetTypeFromString(target) if not targetType: log.error(f"Could not determine target type. Invalid target: {target}") sys.exit(-1) target = target.strip('"') modlist = list() if not args.t and not args.m and not args.u: log.warning( "You didn't specify any modules, types or use case, so all modules will be enabled." ) for m in list(sfModules.keys()): if "__" in m: continue modlist.append(m) signal.signal(signal.SIGINT, handle_abort) # If the user is scanning by type.. # 1. Find modules producing that type if args.t: types = args.t modlist = sf.modulesProducing(types) newmods = deepcopy(modlist) newmodcpy = deepcopy(newmods) # 2. For each type those modules consume, get modules producing while len(newmodcpy) > 0: for etype in sf.eventsToModules(newmodcpy): xmods = sf.modulesProducing([etype]) for mod in xmods: if mod not in modlist: modlist.append(mod) newmods.append(mod) newmodcpy = deepcopy(newmods) newmods = list() # Easier if scanning by module if args.m: modlist = list(filter(None, args.m.split(","))) # Select modules if the user selected usercase if args.u: usecase = args.u[0].upper() + args.u[ 1:] # Make the first Letter Uppercase for mod in sfConfig['__modules__']: if usecase == 'All' or usecase in sfConfig['__modules__'][mod][ 'group']: modlist.append(mod) # Add sfp__stor_stdout to the module list typedata = dbh.eventTypes() types = dict() for r in typedata: types[r[1]] = r[0] sfp__stor_stdout_opts = sfConfig['__modules__']['sfp__stor_stdout']['opts'] sfp__stor_stdout_opts['_eventtypes'] = types if args.f: if args.f and not args.t: log.error("You can only use -f with -t. Use --help for guidance.") sys.exit(-1) sfp__stor_stdout_opts['_showonlyrequested'] = True if args.F: sfp__stor_stdout_opts['_requested'] = args.F.split(",") sfp__stor_stdout_opts['_showonlyrequested'] = True if args.o: if args.o not in ["tab", "csv", "json"]: log.error( "Invalid output format selected. Must be 'tab', 'csv' or 'json'." ) sys.exit(-1) sfp__stor_stdout_opts['_format'] = args.o if args.t: sfp__stor_stdout_opts['_requested'] = args.t.split(",") if args.n: sfp__stor_stdout_opts['_stripnewline'] = True if args.r: sfp__stor_stdout_opts['_showsource'] = True if args.S: sfp__stor_stdout_opts['_maxlength'] = args.S if args.D: sfp__stor_stdout_opts['_csvdelim'] = args.D if args.x: tmodlist = list() modlist = list() xmods = sf.modulesConsuming([targetType]) for mod in xmods: if mod not in modlist: tmodlist.append(mod) # Remove any modules not producing the type requested rtypes = args.t.split(",") for mod in tmodlist: for r in rtypes: if not sfModules[mod]['provides']: continue if r in sfModules[mod].get('provides', []) and mod not in modlist: modlist.append(mod) if len(modlist) == 0: log.error("Based on your criteria, no modules were enabled.") sys.exit(-1) modlist += ["sfp__stor_db", "sfp__stor_stdout"] if sfConfig['__logging']: log.info(f"Modules enabled ({len(modlist)}): {','.join(modlist)}") cfg = sf.configUnserialize(dbh.configGet(), sfConfig) # Debug mode is a variable that gets stored to the DB, so re-apply it if args.debug: cfg['_debug'] = True else: cfg['_debug'] = False # If strict mode is enabled, filter the output from modules. if args.x and args.t: cfg['__outputfilter'] = args.t.split(",") # Prepare scan output headers if args.o == "json": print("[", end='') elif not args.H: delim = "\t" if args.o == "tab": delim = "\t" if args.o == "csv": if args.D: delim = args.D else: delim = "," if args.r: if delim == "\t": headers = '{0:30}{1}{2:45}{3}{4}{5}{6}'.format( "Source", delim, "Type", delim, "Source Data", delim, "Data") else: headers = delim.join(["Source", "Type", "Source Data", "Data"]) else: if delim == "\t": headers = '{0:30}{1}{2:45}{3}{4}'.format( "Source", delim, "Type", delim, "Data") else: headers = delim.join(["Source", "Type", "Data"]) print(headers) # Start running a new scan scanName = target scanId = SpiderFootHelpers.genScanInstanceId() try: p = mp.Process(target=startSpiderFootScanner, args=(loggingQueue, scanName, scanId, target, targetType, modlist, cfg)) p.daemon = True p.start() except BaseException as e: log.error(f"Scan [{scanId}] failed: {e}") sys.exit(-1) # Poll for scan status until completion while True: time.sleep(1) info = dbh.scanInstanceGet(scanId) if not info: continue if info[5] in [ "ERROR-FAILED", "ABORT-REQUESTED", "ABORTED", "FINISHED" ]: if sfConfig['__logging']: log.info(f"Scan completed with status {info[5]}") if args.o == "json": print("]") sys.exit(0) return
def startscan(self, scanname, scantarget, modulelist, typelist, usecase): """Initiate a scan Args: scanname (str): scan name scantarget (str): scan target modulelist (str): comma separated list of modules to use typelist (str): selected modules based on produced event data types usecase (str): selected module group (passive, investigate, footprint, all) Returns: str: start scan status as JSON Raises: HTTPRedirect: redirect to new scan info page """ [scanname, scantarget] = self.cleanUserInput([scanname, scantarget]) if not scanname: if cherrypy.request.headers.get('Accept') and 'application/json' in cherrypy.request.headers.get('Accept'): cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8" return json.dumps(["ERROR", "Incorrect usage: scan name was not specified."]).encode('utf-8') return self.error("Invalid request: scan name was not specified.") if not scantarget: if cherrypy.request.headers.get('Accept') and 'application/json' in cherrypy.request.headers.get('Accept'): cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8" return json.dumps(["ERROR", "Incorrect usage: scan target was not specified."]).encode('utf-8') return self.error("Invalid request: scan target was not specified.") if not typelist and not modulelist and not usecase: if cherrypy.request.headers.get('Accept') and 'application/json' in cherrypy.request.headers.get('Accept'): cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8" return json.dumps(["ERROR", "Incorrect usage: no modules specified for scan."]).encode('utf-8') return self.error("Invalid request: no modules specified for scan.") targetType = SpiderFootHelpers.targetTypeFromString(scantarget) if targetType is None: if cherrypy.request.headers.get('Accept') and 'application/json' in cherrypy.request.headers.get('Accept'): cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8" return json.dumps(["ERROR", "Unrecognised target type."]).encode('utf-8') return self.error("Invalid target type. Could not recognize it as a target SpiderFoot supports.") # Swap the globalscantable for the database handler dbh = SpiderFootDb(self.config) # Snapshot the current configuration to be used by the scan cfg = deepcopy(self.config) sf = SpiderFoot(cfg) modlist = list() # User selected modules if modulelist: modlist = modulelist.replace('module_', '').split(',') # User selected types if len(modlist) == 0 and typelist: typesx = typelist.replace('type_', '').split(',') # 1. Find all modules that produce the requested types modlist = sf.modulesProducing(typesx) newmods = deepcopy(modlist) newmodcpy = deepcopy(newmods) # 2. For each type those modules consume, get modules producing while len(newmodcpy) > 0: for etype in sf.eventsToModules(newmodcpy): xmods = sf.modulesProducing([etype]) for mod in xmods: if mod not in modlist: modlist.append(mod) newmods.append(mod) newmodcpy = deepcopy(newmods) newmods = list() # User selected a use case if len(modlist) == 0 and usecase: for mod in self.config['__modules__']: if usecase == 'all' or usecase in self.config['__modules__'][mod]['group']: modlist.append(mod) # If we somehow got all the way through to here and still don't have any modules selected if not modlist: if cherrypy.request.headers.get('Accept') and 'application/json' in cherrypy.request.headers.get('Accept'): cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8" return json.dumps(["ERROR", "Incorrect usage: no modules specified for scan."]).encode('utf-8') return self.error("Invalid request: no modules specified for scan.") # Add our mandatory storage module if "sfp__stor_db" not in modlist: modlist.append("sfp__stor_db") modlist.sort() # Delete the stdout module in case it crept in if "sfp__stor_stdout" in modlist: modlist.remove("sfp__stor_stdout") # Start running a new scan if targetType in ["HUMAN_NAME", "USERNAME", "BITCOIN_ADDRESS"]: scantarget = scantarget.replace("\"", "") else: scantarget = scantarget.lower() # Start running a new scan scanId = SpiderFootHelpers.genScanInstanceId() try: p = mp.Process(target=SpiderFootScanner, args=(scanname, scanId, scantarget, targetType, modlist, cfg)) p.daemon = True p.start() except Exception as e: self.log.error(f"[-] Scan [{scanId}] failed: {e}") return self.error(f"[-] Scan [{scanId}] failed: {e}") # Wait until the scan has initialized # Check the database for the scan status results while dbh.scanInstanceGet(scanId) is None: self.log.info("Waiting for the scan to initialize...") time.sleep(1) if cherrypy.request.headers.get('Accept') and 'application/json' in cherrypy.request.headers.get('Accept'): cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8" return json.dumps(["SUCCESS", scanId]).encode('utf-8') raise cherrypy.HTTPRedirect(f"{self.docroot}/scaninfo?id={scanId}")