def savesettings(self, allopts, token, configFile=None): """Save settings, also used to completely reset them to default Args: allopts: TBD token: TBD configFile: TBD """ if str(token) != str(self.token): return self.error("Invalid token (%s)" % self.token) if configFile: # configFile seems to get set even if a file isn't uploaded if configFile.file: contents = configFile.file.read() if type(contents) == bytes: contents = contents.decode('utf-8') try: tmp = dict() for line in contents.split("\n"): if "=" not in line: continue opt_array = line.strip().split("=") if len(opt_array) == 1: opt_array[1] = "" tmp[opt_array[0]] = '='.join(opt_array[1:]) allopts = json.dumps(tmp).encode('utf-8') except BaseException as e: return self.error("Failed to parse input file. Was it generated from SpiderFoot? (%s)" % e) # Reset config to default if allopts == "RESET": if self.reset_settings(): raise cherrypy.HTTPRedirect(f"{self.docroot}/opts?updated=1") else: return self.error("Failed to reset settings") # Save settings try: dbh = SpiderFootDb(self.config) useropts = json.loads(allopts) cleanopts = dict() for opt in list(useropts.keys()): cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0] currentopts = deepcopy(self.config) # Make a new config where the user options override # the current system config. sf = SpiderFoot(self.config) self.config = sf.configUnserialize(cleanopts, currentopts) dbh.configSet(sf.configSerialize(self.config)) except Exception as e: return self.error("Processing one or more of your inputs failed: %s" % e) raise cherrypy.HTTPRedirect(f"{self.docroot}/opts?updated=1")
def __init__(self, scanName, scanId, targetValue, targetType, moduleList, globalOpts, start=True): """Initialize SpiderFootScanner object. Args: scanName (str): name of the scan scanId (str): unique ID of the scan targetValue (str): scan target targetType (str): scan target type moduleList (list): list of modules to run globalOpts (dict): scan options start (bool): start the scan immediately Raises: TypeError: arg type was invalid ValueError: arg value was invalid Todo: Eventually change this to be able to control multiple scan instances """ if not isinstance(globalOpts, dict): raise TypeError( f"globalOpts is {type(globalOpts)}; expected dict()") if not globalOpts: raise ValueError("globalOpts is empty") self.__config = deepcopy(globalOpts) self.__dbh = SpiderFootDb(self.__config) if not isinstance(scanName, str): raise TypeError(f"scanName is {type(scanName)}; expected str()") if not scanName: raise ValueError("scanName value is blank") self.__scanName = scanName if not isinstance(scanId, str): raise TypeError(f"scanId is {type(scanId)}; expected str()") if not scanId: raise ValueError("scanId value is blank") if not isinstance(targetValue, str): raise TypeError( f"targetValue is {type(targetValue)}; expected str()") if not targetValue: raise ValueError("targetValue value is blank") self.__targetValue = targetValue if not isinstance(targetType, str): raise TypeError( f"targetType is {type(targetType)}; expected str()") if not targetType: raise ValueError("targetType value is blank") self.__targetType = targetType if not isinstance(moduleList, list): raise TypeError( f"moduleList is {type(moduleList)}; expected list()") if not moduleList: raise ValueError("moduleList is empty") self.__moduleList = moduleList self.__sf = SpiderFoot(self.__config) self.__sf.dbh = self.__dbh # Create a unique ID for this scan in the back-end DB. if scanId: self.__scanId = scanId else: self.__scanId = SpiderFootHelpers.genScanInstanceId() self.__sf.scanId = self.__scanId self.__dbh.scanInstanceCreate(self.__scanId, self.__scanName, self.__targetValue) # Create our target try: self.__target = SpiderFootTarget(self.__targetValue, self.__targetType) except (TypeError, ValueError) as e: self.__sf.status(f"Scan [{self.__scanId}] failed: {e}") self.__setStatus("ERROR-FAILED", None, time.time() * 1000) raise ValueError(f"Invalid target: {e}") # Save the config current set for this scan self.__config['_modulesenabled'] = self.__moduleList self.__dbh.scanConfigSet( self.__scanId, self.__sf.configSerialize(deepcopy(self.__config))) # Process global options that point to other places for data # If a proxy server was specified, set it up proxy_type = self.__config.get('_socks1type') if proxy_type: # TODO: allow DNS lookup to be configurable when using a proxy # - proxy DNS lookup: socks5h:// and socks4a:// # - local DNS lookup: socks5:// and socks4:// if proxy_type == '4': proxy_proto = 'socks4://' elif proxy_type == '5': proxy_proto = 'socks5://' elif proxy_type == 'HTTP': proxy_proto = 'http://' elif proxy_type == 'TOR': proxy_proto = 'socks5h://' else: self.__sf.status( f"Scan [{self.__scanId}] failed: Invalid proxy type: {proxy_type}" ) self.__setStatus("ERROR-FAILED", None, time.time() * 1000) raise ValueError(f"Invalid proxy type: {proxy_type}") proxy_host = self.__config.get('_socks2addr', '') if not proxy_host: self.__sf.status( f"Scan [{self.__scanId}] failed: Proxy type is set ({proxy_type}) but proxy address value is blank" ) self.__setStatus("ERROR-FAILED", None, time.time() * 1000) raise ValueError( f"Proxy type is set ({proxy_type}) but proxy address value is blank" ) proxy_port = int(self.__config.get('_socks3port') or 0) if not proxy_port: if proxy_type in ['4', '5']: proxy_port = 1080 elif proxy_type.upper() == 'HTTP': proxy_port = 8080 elif proxy_type.upper() == 'TOR': proxy_port = 9050 proxy_username = self.__config.get('_socks4user', '') proxy_password = self.__config.get('_socks5pwd', '') if proxy_username or proxy_password: proxy_auth = f"{proxy_username}:{proxy_password}" proxy = f"{proxy_proto}{proxy_auth}@{proxy_host}:{proxy_port}" else: proxy = f"{proxy_proto}{proxy_host}:{proxy_port}" self.__sf.debug(f"Using proxy: {proxy}") self.__sf.socksProxy = proxy else: self.__sf.socksProxy = None # Override the default DNS server if self.__config['_dnsserver']: res = dns.resolver.Resolver() res.nameservers = [self.__config['_dnsserver']] dns.resolver.override_system_resolver(res) else: dns.resolver.restore_system_resolver() # Set the user agent self.__config['_useragent'] = self.__sf.optValueToData( self.__config['_useragent']) # Get internet TLDs tlddata = self.__sf.cacheGet("internet_tlds", self.__config['_internettlds_cache']) # If it wasn't loadable from cache, load it from scratch if tlddata is None: self.__config['_internettlds'] = self.__sf.optValueToData( self.__config['_internettlds']) self.__sf.cachePut("internet_tlds", self.__config['_internettlds']) else: self.__config["_internettlds"] = tlddata.splitlines() self.__setStatus("INITIALIZING", time.time() * 1000, None) self.__sharedThreadPool = SpiderFootThreadPool( threads=self.__config.get("_maxthreads", 3), name='sharedThreadPool') # Used when module threading is enabled self.eventQueue = None if start: self.__startScan()
class SpiderFootScanner(): """SpiderFootScanner object. Attributes: scanId (str): unique ID of the scan status (str): status of the scan """ __scanId = None __status = None __config = None __sf = None __dbh = None __targetValue = None __targetType = None __moduleList = list() __target = None __moduleInstances = dict() __modconfig = dict() __scanName = None def __init__(self, scanName, scanId, targetValue, targetType, moduleList, globalOpts, start=True): """Initialize SpiderFootScanner object. Args: scanName (str): name of the scan scanId (str): unique ID of the scan targetValue (str): scan target targetType (str): scan target type moduleList (list): list of modules to run globalOpts (dict): scan options start (bool): start the scan immediately Raises: TypeError: arg type was invalid ValueError: arg value was invalid Todo: Eventually change this to be able to control multiple scan instances """ if not isinstance(globalOpts, dict): raise TypeError( f"globalOpts is {type(globalOpts)}; expected dict()") if not globalOpts: raise ValueError("globalOpts is empty") self.__config = deepcopy(globalOpts) self.__dbh = SpiderFootDb(self.__config) if not isinstance(scanName, str): raise TypeError(f"scanName is {type(scanName)}; expected str()") if not scanName: raise ValueError("scanName value is blank") self.__scanName = scanName if not isinstance(scanId, str): raise TypeError(f"scanId is {type(scanId)}; expected str()") if not scanId: raise ValueError("scanId value is blank") if not isinstance(targetValue, str): raise TypeError( f"targetValue is {type(targetValue)}; expected str()") if not targetValue: raise ValueError("targetValue value is blank") self.__targetValue = targetValue if not isinstance(targetType, str): raise TypeError( f"targetType is {type(targetType)}; expected str()") if not targetType: raise ValueError("targetType value is blank") self.__targetType = targetType if not isinstance(moduleList, list): raise TypeError( f"moduleList is {type(moduleList)}; expected list()") if not moduleList: raise ValueError("moduleList is empty") self.__moduleList = moduleList self.__sf = SpiderFoot(self.__config) self.__sf.dbh = self.__dbh # Create a unique ID for this scan in the back-end DB. if scanId: self.__scanId = scanId else: self.__scanId = SpiderFootHelpers.genScanInstanceId() self.__sf.scanId = self.__scanId self.__dbh.scanInstanceCreate(self.__scanId, self.__scanName, self.__targetValue) # Create our target try: self.__target = SpiderFootTarget(self.__targetValue, self.__targetType) except (TypeError, ValueError) as e: self.__sf.status(f"Scan [{self.__scanId}] failed: {e}") self.__setStatus("ERROR-FAILED", None, time.time() * 1000) raise ValueError(f"Invalid target: {e}") # Save the config current set for this scan self.__config['_modulesenabled'] = self.__moduleList self.__dbh.scanConfigSet( self.__scanId, self.__sf.configSerialize(deepcopy(self.__config))) # Process global options that point to other places for data # If a proxy server was specified, set it up proxy_type = self.__config.get('_socks1type') if proxy_type: # TODO: allow DNS lookup to be configurable when using a proxy # - proxy DNS lookup: socks5h:// and socks4a:// # - local DNS lookup: socks5:// and socks4:// if proxy_type == '4': proxy_proto = 'socks4://' elif proxy_type == '5': proxy_proto = 'socks5://' elif proxy_type == 'HTTP': proxy_proto = 'http://' elif proxy_type == 'TOR': proxy_proto = 'socks5h://' else: self.__sf.status( f"Scan [{self.__scanId}] failed: Invalid proxy type: {proxy_type}" ) self.__setStatus("ERROR-FAILED", None, time.time() * 1000) raise ValueError(f"Invalid proxy type: {proxy_type}") proxy_host = self.__config.get('_socks2addr', '') if not proxy_host: self.__sf.status( f"Scan [{self.__scanId}] failed: Proxy type is set ({proxy_type}) but proxy address value is blank" ) self.__setStatus("ERROR-FAILED", None, time.time() * 1000) raise ValueError( f"Proxy type is set ({proxy_type}) but proxy address value is blank" ) proxy_port = int(self.__config.get('_socks3port') or 0) if not proxy_port: if proxy_type in ['4', '5']: proxy_port = 1080 elif proxy_type.upper() == 'HTTP': proxy_port = 8080 elif proxy_type.upper() == 'TOR': proxy_port = 9050 proxy_username = self.__config.get('_socks4user', '') proxy_password = self.__config.get('_socks5pwd', '') if proxy_username or proxy_password: proxy_auth = f"{proxy_username}:{proxy_password}" proxy = f"{proxy_proto}{proxy_auth}@{proxy_host}:{proxy_port}" else: proxy = f"{proxy_proto}{proxy_host}:{proxy_port}" self.__sf.debug(f"Using proxy: {proxy}") self.__sf.socksProxy = proxy else: self.__sf.socksProxy = None # Override the default DNS server if self.__config['_dnsserver']: res = dns.resolver.Resolver() res.nameservers = [self.__config['_dnsserver']] dns.resolver.override_system_resolver(res) else: dns.resolver.restore_system_resolver() # Set the user agent self.__config['_useragent'] = self.__sf.optValueToData( self.__config['_useragent']) # Get internet TLDs tlddata = self.__sf.cacheGet("internet_tlds", self.__config['_internettlds_cache']) # If it wasn't loadable from cache, load it from scratch if tlddata is None: self.__config['_internettlds'] = self.__sf.optValueToData( self.__config['_internettlds']) self.__sf.cachePut("internet_tlds", self.__config['_internettlds']) else: self.__config["_internettlds"] = tlddata.splitlines() self.__setStatus("INITIALIZING", time.time() * 1000, None) self.__sharedThreadPool = SpiderFootThreadPool( threads=self.__config.get("_maxthreads", 3), name='sharedThreadPool') # Used when module threading is enabled self.eventQueue = None if start: self.__startScan() @property def scanId(self): return self.__scanId @property def status(self): return self.__status def __setStatus(self, status, started=None, ended=None): """Set the status of the currently running scan (if any). Args: status (str): scan status started (float): timestamp at start of scan ended (float): timestamp at end of scan Raises: TypeError: arg type was invalid ValueError: arg value was invalid """ if not isinstance(status, str): raise TypeError(f"status is {type(status)}; expected str()") if status not in [ "INITIALIZING", "STARTING", "STARTED", "RUNNING", "ABORT-REQUESTED", "ABORTED", "ABORTING", "FINISHED", "ERROR-FAILED" ]: raise ValueError(f"Invalid scan status {status}") self.__status = status self.__dbh.scanInstanceSet(self.__scanId, started, ended, status) def __startScan(self): """Start running a scan. Raises: AssertionError: Never actually raised. """ failed = True try: self.__setStatus("STARTING", time.time() * 1000, None) self.__sf.status( f"Scan [{self.__scanId}] for '{self.__target.targetValue}' initiated." ) self.eventQueue = queue.Queue() self.__sharedThreadPool.start() # moduleList = list of modules the user wants to run self.__sf.debug(f"Loading {len(self.__moduleList)} modules ...") for modName in self.__moduleList: if not modName: continue # Module may have been renamed or removed if modName not in self.__config['__modules__']: self.__sf.error(f"Failed to load module: {modName}") continue try: module = __import__('modules.' + modName, globals(), locals(), [modName]) except ImportError: self.__sf.error(f"Failed to load module: {modName}") continue try: mod = getattr(module, modName)() mod.__name__ = modName except Exception: self.__sf.error( f"Module {modName} initialization failed: {traceback.format_exc()}" ) continue # Set up the module options, scan ID, database handle and listeners try: # Configuration is a combined global config with module-specific options self.__modconfig[modName] = deepcopy( self.__config['__modules__'][modName]['opts']) for opt in list(self.__config.keys()): self.__modconfig[modName][opt] = deepcopy( self.__config[opt]) # clear any listener relationships from the past mod.clearListeners() mod.setScanId(self.__scanId) mod.setSharedThreadPool(self.__sharedThreadPool) mod.setDbh(self.__dbh) mod.setup(self.__sf, self.__modconfig[modName]) except Exception: self.__sf.error( f"Module {modName} initialization failed: {traceback.format_exc()}" ) mod.errorState = True continue # Override the module's local socket module to be the SOCKS one. if self.__config['_socks1type'] != '': try: mod._updateSocket(socket) except Exception as e: self.__sf.error( f"Module {modName} socket setup failed: {e}") continue # Set up event output filters if requested if self.__config['__outputfilter']: try: mod.setOutputFilter(self.__config['__outputfilter']) except Exception as e: self.__sf.error( f"Module {modName} output filter setup failed: {e}" ) continue # Give modules a chance to 'enrich' the original target with aliases of that target. try: newTarget = mod.enrichTarget(self.__target) if newTarget is not None: self.__target = newTarget except Exception as e: self.__sf.error( f"Module {modName} target enrichment failed: {e}") continue # Register the target with the module try: mod.setTarget(self.__target) except Exception as e: self.__sf.error( f"Module {modName} failed to set target '{self.__target}': {e}" ) continue # Set up the outgoing event queue try: mod.outgoingEventQueue = self.eventQueue mod.incomingEventQueue = queue.Queue() except Exception as e: self.__sf.error( f"Module {modName} event queue setup failed: {e}") continue self.__moduleInstances[modName] = mod self.__sf.status(f"{modName} module loaded.") self.__sf.debug( f"Scan [{self.__scanId}] loaded {len(self.__moduleInstances)} modules." ) if not self.__moduleInstances: self.__setStatus("ERROR-FAILED", None, time.time() * 1000) self.__dbh.close() return # sort modules by priority self.__moduleInstances = OrderedDict( sorted(self.__moduleInstances.items(), key=lambda m: m[-1]._priority)) # Now we are ready to roll.. self.__setStatus("RUNNING") # Create a pseudo module for the root event to originate from psMod = SpiderFootPlugin() psMod.__name__ = "SpiderFoot UI" psMod.setTarget(self.__target) psMod.setDbh(self.__dbh) psMod.clearListeners() psMod.outgoingEventQueue = self.eventQueue psMod.incomingEventQueue = queue.Queue() # Create the "ROOT" event which un-triggered modules will link events to rootEvent = SpiderFootEvent("ROOT", self.__targetValue, "", None) psMod.notifyListeners(rootEvent) firstEvent = SpiderFootEvent(self.__targetType, self.__targetValue, "SpiderFoot UI", rootEvent) psMod.notifyListeners(firstEvent) # Special case.. check if an INTERNET_NAME is also a domain if self.__targetType == 'INTERNET_NAME': if self.__sf.isDomain(self.__targetValue, self.__config['_internettlds']): firstEvent = SpiderFootEvent('DOMAIN_NAME', self.__targetValue, "SpiderFoot UI", rootEvent) psMod.notifyListeners(firstEvent) # If in interactive mode, loop through this shared global variable # waiting for inputs, and process them until my status is set to # FINISHED. # Check in case the user requested to stop the scan between modules # initializing scanstatus = self.__dbh.scanInstanceGet(self.__scanId) if scanstatus and scanstatus[5] == "ABORT-REQUESTED": raise AssertionError("ABORT-REQUESTED") # start threads self.waitForThreads() failed = False except (KeyboardInterrupt, AssertionError): self.__sf.status(f"Scan [{self.__scanId}] aborted.") self.__setStatus("ABORTED", None, time.time() * 1000) except BaseException as e: exc_type, exc_value, exc_traceback = sys.exc_info() self.__sf.error( f"Unhandled exception ({e.__class__.__name__}) encountered during scan." + "Please report this as a bug: " + repr( traceback.format_exception(exc_type, exc_value, exc_traceback))) self.__sf.status(f"Scan [{self.__scanId}] failed: {e}") self.__setStatus("ERROR-FAILED", None, time.time() * 1000) finally: if not failed: self.runCorrelations() self.__sf.status(f"Scan [{self.__scanId}] completed.") self.__setStatus("FINISHED", None, time.time() * 1000) self.__dbh.close() def runCorrelations(self): self.__sf.status( f"Running {len(self.__config['__correlationrules__'])} correlation rules." ) ruleset = dict() for rule in self.__config['__correlationrules__']: ruleset[rule['id']] = rule['rawYaml'] corr = SpiderFootCorrelator(self.__dbh, ruleset, self.__scanId) corr.run_correlations() def waitForThreads(self): counter = 0 try: if not self.eventQueue: return # start one thread for each module for mod in self.__moduleInstances.values(): mod.start() final_passes = 3 # watch for newly-generated events while True: # log status of threads every 10 iterations log_status = counter % 10 == 0 counter += 1 if log_status: scanstatus = self.__dbh.scanInstanceGet(self.__scanId) if scanstatus and scanstatus[5] == "ABORT-REQUESTED": raise AssertionError("ABORT-REQUESTED") try: sfEvent = self.eventQueue.get_nowait() self.__sf.debug( f"waitForThreads() got event, {sfEvent.eventType}, from eventQueue." ) except queue.Empty: # check if we're finished if self.threadsFinished(log_status): sleep(.1) # but are we really? if self.threadsFinished(log_status): if final_passes < 1: break # Trigger module.finished() for mod in self.__moduleInstances.values(): if not mod.errorState and mod.incomingEventQueue is not None: mod.incomingEventQueue.put('FINISHED') sleep(.1) while not self.threadsFinished(log_status): log_status = counter % 100 == 0 counter += 1 sleep(.01) final_passes -= 1 else: # save on CPU sleep(.1) continue if not isinstance(sfEvent, SpiderFootEvent): raise TypeError( f"sfEvent is {type(sfEvent)}; expected SpiderFootEvent" ) # for every module for mod in self.__moduleInstances.values(): # if it's been aborted if mod._stopScanning: # break out of the while loop raise AssertionError(f"{mod.__name__} requested stop") # send it the new event if applicable if not mod.errorState and mod.incomingEventQueue is not None: watchedEvents = mod.watchedEvents() if sfEvent.eventType in watchedEvents or "*" in watchedEvents: mod.incomingEventQueue.put(deepcopy(sfEvent)) finally: # tell the modules to stop for mod in self.__moduleInstances.values(): mod._stopScanning = True self.__sharedThreadPool.shutdown(wait=True) def threadsFinished(self, log_status=False): if self.eventQueue is None: return True modules_waiting = dict() for m in self.__moduleInstances.values(): try: if m.incomingEventQueue is not None: modules_waiting[m.__name__] = m.incomingEventQueue.qsize() except Exception: with suppress(Exception): m.errorState = True modules_waiting = sorted(modules_waiting.items(), key=lambda x: x[-1], reverse=True) modules_running = [] for m in self.__moduleInstances.values(): try: if m.running: modules_running.append(m.__name__) except Exception: with suppress(Exception): m.errorState = True modules_errored = [] for m in self.__moduleInstances.values(): try: if m.errorState: modules_errored.append(m.__name__) except Exception: with suppress(Exception): m.errorState = True queues_empty = [qsize == 0 for m, qsize in modules_waiting] for mod in self.__moduleInstances.values(): if mod.errorState and mod.incomingEventQueue is not None: self.__sf.debug( f"Clearing and unsetting incomingEventQueue for errored module {mod.__name__}." ) with suppress(Exception): while 1: mod.incomingEventQueue.get_nowait() mod.incomingEventQueue = None if not modules_running and not queues_empty: self.__sf.debug("Clearing queues for stalled/aborted modules.") for mod in self.__moduleInstances.values(): try: while True: mod.incomingEventQueue.get_nowait() except Exception: pass if log_status: events_queued = ", ".join([ f"{mod}: {qsize:,}" for mod, qsize in modules_waiting[:5] if qsize > 0 ]) if not events_queued: events_queued = 'None' self.__sf.debug( f"Events queued: {sum([m[-1] for m in modules_waiting]):,} ({events_queued})" ) if modules_running: self.__sf.debug( f"Modules running: {len(modules_running):,} ({', '.join(modules_running)})" ) if modules_errored: self.__sf.debug( f"Modules errored: {len(modules_errored):,} ({', '.join(modules_errored)})" ) if all(queues_empty) and not modules_running: return True return False
def __init__(self, scanName, scanId, targetValue, targetType, moduleList, globalOpts, start=True): """Initialize SpiderFootScanner object. Args: scanName (str): name of the scan scanId (str): unique ID of the scan targetValue (str): scan target targetType (str): scan target type moduleList (list): list of modules to run globalOpts (dict): scan options start (bool): start the scan immediately Raises: TypeError: arg type was invalid ValueError: arg value was invalid Todo: Eventually change this to be able to control multiple scan instances """ if not isinstance(globalOpts, dict): raise TypeError(f"globalOpts is {type(globalOpts)}; expected dict()") if not globalOpts: raise ValueError("globalOpts is empty") self.__config = deepcopy(globalOpts) self.__dbh = SpiderFootDb(self.__config) if not isinstance(scanName, str): raise TypeError(f"scanName is {type(scanName)}; expected str()") if not scanName: raise ValueError("scanName value is blank") self.__scanName = scanName if not isinstance(scanId, str): raise TypeError(f"scanId is {type(scanId)}; expected str()") if not scanId: raise ValueError("scanId value is blank") if not isinstance(targetValue, str): raise TypeError(f"targetValue is {type(targetValue)}; expected str()") if not targetValue: raise ValueError("targetValue value is blank") self.__targetValue = targetValue if not isinstance(targetType, str): raise TypeError(f"targetType is {type(targetType)}; expected str()") if not targetType: raise ValueError("targetType value is blank") self.__targetType = targetType if not isinstance(moduleList, list): raise TypeError(f"moduleList is {type(moduleList)}; expected list()") if not moduleList: raise ValueError("moduleList is empty") self.__moduleList = moduleList self.__sf = SpiderFoot(self.__config) self.__sf.dbh = self.__dbh # Create a unique ID for this scan in the back-end DB. if not isinstance(scanId, str): raise TypeError(f"scanId is {type(scanId)}; expected str()") if scanId: self.__scanId = scanId else: self.__scanId = self.__sf.genScanInstanceId() self.__sf.scanId = self.__scanId self.__dbh.scanInstanceCreate(self.__scanId, self.__scanName, self.__targetValue) # Create our target try: self.__target = SpiderFootTarget(self.__targetValue, self.__targetType) except (TypeError, ValueError) as e: self.__sf.status(f"Scan [{self.__scanId}] failed: {e}") self.__setStatus("ERROR-FAILED", None, time.time() * 1000) raise ValueError(f"Invalid target: {e}") # Save the config current set for this scan self.__config['_modulesenabled'] = self.__moduleList self.__dbh.scanConfigSet(self.__scanId, self.__sf.configSerialize(deepcopy(self.__config))) # Process global options that point to other places for data # If a SOCKS server was specified, set it up if self.__config['_socks1type']: socksAddr = self.__config['_socks2addr'] socksPort = int(self.__config['_socks3port']) socksUsername = self.__config['_socks4user'] or '' socksPassword = self.__config['_socks5pwd'] or '' proxy = f"{socksAddr}:{socksPort}" if socksUsername or socksPassword: proxy = "%s:%s@%s" % (socksUsername, socksPassword, proxy) if self.__config['_socks1type'] == '4': proxy = 'socks4://' + proxy elif self.__config['_socks1type'] == '5': proxy = 'socks5://' + proxy elif self.__config['_socks1type'] == 'HTTP': proxy = 'http://' + proxy elif self.__config['_socks1type'] == 'TOR': proxy = 'socks5h://' + proxy else: raise ValueError(f"Invalid SOCKS proxy type: {self.__config['_socks1ttype']}") self.__sf.debug(f"SOCKS: {socksAddr}:{socksPort} ({socksUsername}:{socksPassword})") self.__sf.socksProxy = proxy else: self.__sf.socksProxy = None # Override the default DNS server if self.__config['_dnsserver']: res = dns.resolver.Resolver() res.nameservers = [self.__config['_dnsserver']] dns.resolver.override_system_resolver(res) else: dns.resolver.restore_system_resolver() # Set the user agent self.__config['_useragent'] = self.__sf.optValueToData(self.__config['_useragent']) # Get internet TLDs tlddata = self.__sf.cacheGet("internet_tlds", self.__config['_internettlds_cache']) # If it wasn't loadable from cache, load it from scratch if tlddata is None: self.__config['_internettlds'] = self.__sf.optValueToData(self.__config['_internettlds']) self.__sf.cachePut("internet_tlds", self.__config['_internettlds']) else: self.__config["_internettlds"] = tlddata.splitlines() self.__setStatus("INITIALIZING", time.time() * 1000, None) if start: self.__startScan()
class SpiderFootScanner(): """SpiderFootScanner object. Attributes: scanId (str): unique ID of the scan status (str): status of the scan """ __scanId = None __status = None __config = None __sf = None __dbh = None __targetValue = None __targetType = None __moduleList = list() __target = None __moduleInstances = dict() __modconfig = dict() __scanName = None def __init__(self, scanName, scanId, targetValue, targetType, moduleList, globalOpts, start=True): """Initialize SpiderFootScanner object. Args: scanName (str): name of the scan scanId (str): unique ID of the scan targetValue (str): scan target targetType (str): scan target type moduleList (list): list of modules to run globalOpts (dict): scan options start (bool): start the scan immediately Raises: TypeError: arg type was invalid ValueError: arg value was invalid Todo: Eventually change this to be able to control multiple scan instances """ if not isinstance(globalOpts, dict): raise TypeError(f"globalOpts is {type(globalOpts)}; expected dict()") if not globalOpts: raise ValueError("globalOpts is empty") self.__config = deepcopy(globalOpts) self.__dbh = SpiderFootDb(self.__config) if not isinstance(scanName, str): raise TypeError(f"scanName is {type(scanName)}; expected str()") if not scanName: raise ValueError("scanName value is blank") self.__scanName = scanName if not isinstance(scanId, str): raise TypeError(f"scanId is {type(scanId)}; expected str()") if not scanId: raise ValueError("scanId value is blank") if not isinstance(targetValue, str): raise TypeError(f"targetValue is {type(targetValue)}; expected str()") if not targetValue: raise ValueError("targetValue value is blank") self.__targetValue = targetValue if not isinstance(targetType, str): raise TypeError(f"targetType is {type(targetType)}; expected str()") if not targetType: raise ValueError("targetType value is blank") self.__targetType = targetType if not isinstance(moduleList, list): raise TypeError(f"moduleList is {type(moduleList)}; expected list()") if not moduleList: raise ValueError("moduleList is empty") self.__moduleList = moduleList self.__sf = SpiderFoot(self.__config) self.__sf.dbh = self.__dbh # Create a unique ID for this scan in the back-end DB. if not isinstance(scanId, str): raise TypeError(f"scanId is {type(scanId)}; expected str()") if scanId: self.__scanId = scanId else: self.__scanId = self.__sf.genScanInstanceId() self.__sf.scanId = self.__scanId self.__dbh.scanInstanceCreate(self.__scanId, self.__scanName, self.__targetValue) # Create our target try: self.__target = SpiderFootTarget(self.__targetValue, self.__targetType) except (TypeError, ValueError) as e: self.__sf.status(f"Scan [{self.__scanId}] failed: {e}") self.__setStatus("ERROR-FAILED", None, time.time() * 1000) raise ValueError(f"Invalid target: {e}") # Save the config current set for this scan self.__config['_modulesenabled'] = self.__moduleList self.__dbh.scanConfigSet(self.__scanId, self.__sf.configSerialize(deepcopy(self.__config))) # Process global options that point to other places for data # If a SOCKS server was specified, set it up if self.__config['_socks1type']: socksAddr = self.__config['_socks2addr'] socksPort = int(self.__config['_socks3port']) socksUsername = self.__config['_socks4user'] or '' socksPassword = self.__config['_socks5pwd'] or '' proxy = f"{socksAddr}:{socksPort}" if socksUsername or socksPassword: proxy = "%s:%s@%s" % (socksUsername, socksPassword, proxy) if self.__config['_socks1type'] == '4': proxy = 'socks4://' + proxy elif self.__config['_socks1type'] == '5': proxy = 'socks5://' + proxy elif self.__config['_socks1type'] == 'HTTP': proxy = 'http://' + proxy elif self.__config['_socks1type'] == 'TOR': proxy = 'socks5h://' + proxy else: raise ValueError(f"Invalid SOCKS proxy type: {self.__config['_socks1ttype']}") self.__sf.debug(f"SOCKS: {socksAddr}:{socksPort} ({socksUsername}:{socksPassword})") self.__sf.socksProxy = proxy else: self.__sf.socksProxy = None # Override the default DNS server if self.__config['_dnsserver']: res = dns.resolver.Resolver() res.nameservers = [self.__config['_dnsserver']] dns.resolver.override_system_resolver(res) else: dns.resolver.restore_system_resolver() # Set the user agent self.__config['_useragent'] = self.__sf.optValueToData(self.__config['_useragent']) # Get internet TLDs tlddata = self.__sf.cacheGet("internet_tlds", self.__config['_internettlds_cache']) # If it wasn't loadable from cache, load it from scratch if tlddata is None: self.__config['_internettlds'] = self.__sf.optValueToData(self.__config['_internettlds']) self.__sf.cachePut("internet_tlds", self.__config['_internettlds']) else: self.__config["_internettlds"] = tlddata.splitlines() self.__setStatus("INITIALIZING", time.time() * 1000, None) if start: self.__startScan() @property def scanId(self): return self.__scanId @property def status(self): return self.__status def __setStatus(self, status, started=None, ended=None): """Set the status of the currently running scan (if any). Args: status (str): scan status started (float): timestamp at start of scan ended (float): timestamp at end of scan Raises: TypeError: arg type was invalid ValueError: arg value was invalid """ if not isinstance(status, str): raise TypeError(f"status is {type(status)}; expected str()") if status not in [ "INITIALIZING", "STARTING", "STARTED", "RUNNING", "ABORT-REQUESTED", "ABORTED", "ABORTING", "FINISHED", "ERROR-FAILED" ]: raise ValueError(f"Invalid scan status {status}") self.__status = status self.__dbh.scanInstanceSet(self.__scanId, started, ended, status) def __startScan(self): """Start running a scan.""" aborted = False self.__setStatus("STARTING", time.time() * 1000, None) self.__sf.status(f"Scan [{self.__scanId}] initiated.") try: # moduleList = list of modules the user wants to run for modName in self.__moduleList: if modName == '': continue try: module = __import__('modules.' + modName, globals(), locals(), [modName]) except ImportError: self.__sf.error(f"Failed to load module: {modName}") continue mod = getattr(module, modName)() mod.__name__ = modName # Module may have been renamed or removed if modName not in self.__config['__modules__']: continue # Set up the module # Configuration is a combined global config with module-specific options self.__modconfig[modName] = deepcopy(self.__config['__modules__'][modName]['opts']) for opt in list(self.__config.keys()): self.__modconfig[modName][opt] = deepcopy(self.__config[opt]) mod.clearListeners() # clear any listener relationships from the past mod.setup(self.__sf, self.__modconfig[modName]) mod.setDbh(self.__dbh) mod.setScanId(self.__scanId) # Give modules a chance to 'enrich' the original target with # aliases of that target. newTarget = mod.enrichTarget(self.__target) if newTarget is not None: self.__target = newTarget self.__moduleInstances[modName] = mod # Override the module's local socket module # to be the SOCKS one. if self.__config['_socks1type'] != '': mod._updateSocket(socket) # Set up event output filters if requested if self.__config['__outputfilter']: mod.setOutputFilter(self.__config['__outputfilter']) self.__sf.status(modName + " module loaded.") # Register listener modules and then start all modules sequentially for module in list(self.__moduleInstances.values()): # Register the target with the module module.setTarget(self.__target) for listenerModule in list(self.__moduleInstances.values()): # Careful not to register twice or you will get duplicate events if listenerModule in module._listenerModules: continue # Note the absence of a check for whether a module can register # to itself. That is intentional because some modules will # act on their own notifications (e.g. sfp_dns)! if listenerModule.watchedEvents() is not None: module.registerListener(listenerModule) # Now we are ready to roll.. self.__setStatus("RUNNING") # Create a pseudo module for the root event to originate from psMod = SpiderFootPlugin() psMod.__name__ = "SpiderFoot UI" psMod.setTarget(self.__target) psMod.setDbh(self.__dbh) psMod.clearListeners() for mod in list(self.__moduleInstances.values()): if mod.watchedEvents() is not None: psMod.registerListener(mod) # Create the "ROOT" event which un-triggered modules will link events to rootEvent = SpiderFootEvent("ROOT", self.__targetValue, "", None) psMod.notifyListeners(rootEvent) firstEvent = SpiderFootEvent(self.__targetType, self.__targetValue, "SpiderFoot UI", rootEvent) psMod.notifyListeners(firstEvent) # Special case.. check if an INTERNET_NAME is also a domain if self.__targetType == 'INTERNET_NAME': if self.__sf.isDomain(self.__targetValue, self.__config['_internettlds']): firstEvent = SpiderFootEvent('DOMAIN_NAME', self.__targetValue, "SpiderFoot UI", rootEvent) psMod.notifyListeners(firstEvent) # If in interactive mode, loop through this shared global variable # waiting for inputs, and process them until my status is set to # FINISHED. # Check in case the user requested to stop the scan between modules # initializing for module in list(self.__moduleInstances.values()): if module.checkForStop(): self.__setStatus('ABORTING') aborted = True break if aborted: self.__sf.status(f"Scan [{self.__scanId}] aborted.") self.__setStatus("ABORTED", None, time.time() * 1000) else: self.__sf.status(f"Scan [{self.__scanId}] completed.") self.__setStatus("FINISHED", None, time.time() * 1000) except BaseException as e: exc_type, exc_value, exc_traceback = sys.exc_info() self.__sf.error(f"Unhandled exception ({e.__class__.__name__}) encountered during scan." + "Please report this as a bug: " + repr(traceback.format_exception(exc_type, exc_value, exc_traceback))) self.__sf.status(f"Scan [{self.__scanId}] failed: {e}") self.__setStatus("ERROR-FAILED", None, time.time() * 1000) self.__dbh.close()
def start_scan(sfConfig, sfModules, args): global dbh global scanId dbh = SpiderFootDb(sfConfig, init=True) sf = SpiderFoot(sfConfig) if args.modules: log.info("Modules available:") for m in sorted(sfModules.keys()): if "__" in m: continue print(('{0:25} {1}'.format(m, sfModules[m]['descr']))) sys.exit(0) if args.types: log.info("Types available:") typedata = dbh.eventTypes() types = dict() for r in typedata: types[r[1]] = r[0] for t in sorted(types.keys()): print(('{0:45} {1}'.format(t, types[t]))) sys.exit(0) if not args.s: log.error("You must specify a target when running in scan mode. Try --help for guidance.") sys.exit(-1) if args.x and not args.t: log.error("-x can only be used with -t. Use --help for guidance.") sys.exit(-1) if args.x and args.m: log.error("-x can only be used with -t and not with -m. Use --help for guidance.") sys.exit(-1) if args.r and (args.o and args.o not in ["tab", "csv"]): log.error("-r can only be used when your output format is tab or csv.") sys.exit(-1) if args.H and (args.o and args.o not in ["tab", "csv"]): log.error("-H can only be used when your output format is tab or csv.") sys.exit(-1) if args.D and args.o != "csv": log.error("-D can only be used when using the csv output format.") sys.exit(-1) target = args.s # Usernames and names - quoted on the commandline - won't have quotes, # so add them. if " " in target: target = f"\"{target}\"" if "." not in target and not target.startswith("+") and '"' not in target: target = f"\"{target}\"" targetType = sf.targetType(target) if not targetType: log.error(f"Could not determine target type. Invalid target: {target}") sys.exit(-1) target = target.strip('"') modlist = list() if not args.t and not args.m: log.warning("You didn't specify any modules or types, so all will be enabled.") for m in list(sfModules.keys()): if "__" in m: continue modlist.append(m) signal.signal(signal.SIGINT, handle_abort) # If the user is scanning by type.. # 1. Find modules producing that type if args.t: types = args.t modlist = sf.modulesProducing(types) newmods = deepcopy(modlist) newmodcpy = deepcopy(newmods) # 2. For each type those modules consume, get modules producing while len(newmodcpy) > 0: for etype in sf.eventsToModules(newmodcpy): xmods = sf.modulesProducing([etype]) for mod in xmods: if mod not in modlist: modlist.append(mod) newmods.append(mod) newmodcpy = deepcopy(newmods) newmods = list() # Easier if scanning by module if args.m: modlist = list(filter(None, args.m.split(","))) # Add sfp__stor_stdout to the module list typedata = dbh.eventTypes() types = dict() for r in typedata: types[r[1]] = r[0] sfp__stor_stdout_opts = sfConfig['__modules__']['sfp__stor_stdout']['opts'] sfp__stor_stdout_opts['_eventtypes'] = types if args.f: if args.f and not args.t: log.error("You can only use -f with -t. Use --help for guidance.") sys.exit(-1) sfp__stor_stdout_opts['_showonlyrequested'] = True if args.F: sfp__stor_stdout_opts['_requested'] = args.F.split(",") sfp__stor_stdout_opts['_showonlyrequested'] = True if args.o: sfp__stor_stdout_opts['_format'] = args.o if args.t: sfp__stor_stdout_opts['_requested'] = args.t.split(",") if args.n: sfp__stor_stdout_opts['_stripnewline'] = True if args.r: sfp__stor_stdout_opts['_showsource'] = True if args.S: sfp__stor_stdout_opts['_maxlength'] = args.S if args.D: sfp__stor_stdout_opts['_csvdelim'] = args.D if args.x: tmodlist = list() modlist = list() xmods = sf.modulesConsuming([targetType]) for mod in xmods: if mod not in modlist: tmodlist.append(mod) # Remove any modules not producing the type requested rtypes = args.t.split(",") for mod in tmodlist: for r in rtypes: if not sfModules[mod]['provides']: continue if r in sfModules[mod].get('provides', []) and mod not in modlist: modlist.append(mod) if len(modlist) == 0: log.error("Based on your criteria, no modules were enabled.") sys.exit(-1) modlist += ["sfp__stor_db", "sfp__stor_stdout"] # Run the scan if sfConfig['__logging']: log.info(f"Modules enabled ({len(modlist)}): {','.join(modlist)}") cfg = sf.configUnserialize(dbh.configGet(), sfConfig) # Debug mode is a variable that gets stored to the DB, so re-apply it if args.debug: cfg['_debug'] = True else: cfg['_debug'] = False # If strict mode is enabled, filter the output from modules. if args.x and args.t: cfg['__outputfilter'] = args.t.split(",") if args.o == "json": print("[", end='') # Start running a new scan scanName = target scanId = sf.genScanInstanceId() try: p = mp.Process(target=SpiderFootScanner, args=(scanName, scanId, target, targetType, modlist, cfg)) p.daemon = True p.start() except BaseException as e: log.error(f"Scan [{scanId}] failed: {e}") sys.exit(-1) # If field headers weren't disabled, print them if not args.H and args.o != "json": if args.D: delim = args.D else: if args.o in ["tab", None]: delim = "\t" if args.o == "csv": delim = "," if not args.r: if delim != "\t": print(delim.join(["Source", "Type", "Data"])) else: print('{0:30}{1}{2:45}{3}{4}'.format("Source", delim, "Type", delim, "Data")) else: if delim != "\t": print(delim.join(["Source", "Type", "Source Data", "Data"])) else: print('{0:30}{1}{2:45}{3}{4}{5}{6}'.format("Source", delim, "Type", delim, "Source Data", delim, "Data")) while True: time.sleep(1) info = dbh.scanInstanceGet(scanId) if not info: continue if info[5] in ["ERROR-FAILED", "ABORT-REQUESTED", "ABORTED", "FINISHED"]: if sfConfig['__logging']: log.info(f"Scan completed with status {info[5]}") if args.o == "json": print("]") sys.exit(0) return None