def test_get_equivalents_should_return_a_list(self): """ Test _getEquivalents(self, typeName) """ target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) equivalents = target._getEquivalents(target_type) self.assertEqual(list, type(equivalents))
def test_matches_should_return_a_boolean(self): """ Test matches(self, value, includeParents=False, includeChildren=True) """ target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) matches = target.matches(None) self.assertEqual(bool, type(matches))
def test_set_alias(self): """ Test setAlias(self, value, typeName) """ target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) set_alias = target.setAlias(None, None) self.assertEqual('TBD', 'TBD')
def test_get_addresses(self): """ Test getAddresses(self) """ target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) target.getAddresses() self.assertEqual('TBD', 'TBD')
def test_get_names_should_return_a_list(self): """ Test getNames(self) """ target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) names = target.getNames() self.assertEqual(list, type(names))
def test_matches(self): """ Test matches(self, value, includeParents=False, includeChildren=True) """ target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) target.matches(None) self.assertEqual('TBD', 'TBD')
def test_init_supported_target_types(self): """ Test __init__(self, targetValue, typeName) """ target_value = 'example target value' for target_type in self.valid_target_types: with self.subTest(target_type=target_type): target = SpiderFootTarget(target_value, target_type) self.assertEqual(SpiderFootTarget, type(target)) self.assertEqual(target.getType(), target_type) self.assertEqual(target.getValue(), target_value)
def test_processHost_should_return_SpiderFootEvent(self): """ Test processHost(self, host, parentEvent, affiliate=None) """ sf = SpiderFoot(self.default_options) module = sfp_dnsresolve() module.setup(sf, dict()) target_value = '127.0.0.1' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = 'example module' source_event = '' parent_event = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.processHost("127.0.0.1", parent_event, None) self.assertIsInstance(result, SpiderFootEvent) self.assertIsNone(result)
def test_processDomain_should_return_None(self): """ Test processDomain(self, domainName, parentEvent, affil=False, host=None) Todo: review: why should this return None? """ sf = SpiderFoot(self.default_options) module = sfp_dnsresolve() module.setup(sf, dict()) target_value = 'example.local' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = 'example module' source_event = '' parent_event = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.processDomain('www.example.local', parent_event, None, None) self.assertIsNone(result)
def test_init_unsupported_target_type_should_raise(self): """ Test __init__(self, targetValue, typeName) """ with self.assertRaises(ValueError) as cm: target = SpiderFootTarget('example target value', 'example target type')
def test_handleEvent(self): """ Test handleEvent(self, event) """ sf = SpiderFoot(self.default_options) module = sfp_iban() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = 'sfp_test_module' source_event = '' source_event = SpiderFootEvent(event_type, event_data, event_module, source_event) event_type = 'TARGET_WEB_CONTENT' event_data = 'example data' event_module = 'sfp_test_module' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result) self.assertEqual('TBD', 'TBD') self.assertIsNone(result)
def test_target_aliases_attribute_should_return_a_list(self): """ Test targetAliases attribute """ target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) self.assertIsInstance(target.targetAliases, list)
def test_resolve_targets_should_return_list(self): """ Test resolveTargets(self, target, validateReverse) """ sf = SpiderFoot(self.default_options) target = SpiderFootTarget("spiderfoot.net", "INTERNET_NAME") resolve_targets = sf.resolveTargets(target, False) self.assertIsInstance(resolve_targets, list) self.assertIn('spiderfoot.net', resolve_targets)
def test_target_value_attribute_should_return_a_string(self): """ Test targetValue attribute """ target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) self.assertIsInstance(target.targetValue, str) self.assertEqual(target_value, target.targetValue)
def test_init_unsupported_target_type_should_exit(self): """ Test __init__(self, targetValue, typeName) """ target_value = 'example target value' with self.assertRaises(SystemExit) as cm: target_type = 'example target type' target = SpiderFootTarget(target_value, target_type) self.assertEqual(cm.exception.code, -1)
def test_set_target_should_set_a_target(self): """ Test setTarget(self, target) """ sfp = SpiderFootPlugin() target = SpiderFootTarget("spiderfoot.net", "INTERNET_NAME") sfp.setTarget(target) get_target = sfp.getTarget().targetValue self.assertIsInstance(get_target, str) self.assertEqual("spiderfoot.net", get_target)
def test_resolve_targets_should_return_list(self): """ Test resolveTargets(self, target, validateReverse) """ sf = SpiderFoot(self.default_options) invalid_types = [None, "", list()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): resolve_targets = sf.resolveTargets(invalid_type, False) self.assertIsInstance(resolve_targets, list) target = SpiderFootTarget("spiderfoot.net", "INTERNET_NAME") resolve_targets = sf.resolveTargets(target, False) self.assertIsInstance(resolve_targets, list) self.assertIn('spiderfoot.net', resolve_targets) target = SpiderFootTarget("1.1.1.1", "IP_ADDRESS") resolve_targets = sf.resolveTargets(target, False) self.assertIsInstance(resolve_targets, list) self.assertIn('1.1.1.1', resolve_targets)
def test_enrichTarget(self): """ Test enrichTarget(self, target) """ sf = SpiderFoot(self.default_options) module = sfp_dnsresolve() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) result = module.enrichTarget(target) self.assertIsInstance(result, SpiderFootTarget)
def test_handleEvent(self): """ Test handleEvent(self, event) """ sf = SpiderFoot(self.default_options) module = sfp_company() module.setup(sf, dict()) target_value = 'example target value' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt)
def test_handleEvent(self): """ Test handleEvent(self, event) """ sf = SpiderFoot(self.default_options) module = sfp_hostio() module.setup(sf, dict()) target_value = 'example target value' target_type = 'DOMAIN_NAME' target = SpiderFootTarget(target_value, target_type) module.setTarget(target) event_type = 'ROOT' event_data = 'example data' event_module = '' source_event = '' evt = SpiderFootEvent(event_type, event_data, event_module, source_event) result = module.handleEvent(evt) self.assertIsNone(result)
def test_init_invalid_target_value_should_raise(self): """ Test __init__(self, targetValue, typeName) """ with self.assertRaises(TypeError) as cm: target = SpiderFootTarget(None, 'IP_ADDRESS')
def __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts, start=True): """Initialize SpiderFootScanner object. Args: scanName (str): name of the scan scanId (str): unique ID of the scan scanTarget (str): scan target targetType (str): scan target type moduleList (list): list of modules to run globalOpts (dict): scan options start (bool): start the scan immediately Returns: None """ if not isinstance(scanName, str): raise TypeError("scanName is %s; expected str()" % type(scanName)) if not isinstance(scanId, str): raise TypeError("scanId is %s; expected str()" % type(scanId)) if not isinstance(scanTarget, str): raise TypeError("scanTarget is %s; expected str()" % type(scanTarget)) if not isinstance(targetType, str): raise TypeError("targetType is %s; expected str()" % type(targetType)) if not isinstance(moduleList, list): raise TypeError("moduleList is %s; expected list()" % type(moduleList)) if not isinstance(globalOpts, dict): raise TypeError("globalOpts is %s; expected dict()" % type(globalOpts)) if not globalOpts: raise ValueError("globalOpts is empty") self.moduleInstances = dict() self.config = deepcopy(globalOpts) self.sf = SpiderFoot(self.config) self.dbh = SpiderFootDb(self.config) self.targetValue = scanTarget self.targetType = targetType self.moduleList = moduleList self.modconfig = dict() self.scanName = scanName self.sf.setDbh(self.dbh) # Create a unique ID for this scan and create it in the back-end DB. self.scanId = scanId self.sf.setGUID(self.scanId) self.dbh.scanInstanceCreate(self.scanId, self.scanName, self.targetValue) # Create our target try: self.target = SpiderFootTarget(self.targetValue, self.targetType) except ValueError as e: self.sf.status("Scan [%s] failed: %s" % (self.scanId, e)) self.setStatus("ERROR-FAILED", None, time.time() * 1000) raise ValueError("Invalid target: %s" % e) except TypeError as e: self.sf.status("Scan [%s] failed: %s" % (self.scanId, e)) self.setStatus("ERROR-FAILED", None, time.time() * 1000) raise TypeError("Invalid target: %s" % e) # Save the config current set for this scan self.config['_modulesenabled'] = self.moduleList self.dbh.scanConfigSet(self.scanId, self.sf.configSerialize(deepcopy(self.config))) self.setStatus("INITIALIZING", time.time() * 1000, None) if start: self.startScan()
def __init__(self, scanName, scanId, targetValue, targetType, moduleList, globalOpts, start=True): """Initialize SpiderFootScanner object. Args: scanName (str): name of the scan scanId (str): unique ID of the scan targetValue (str): scan target targetType (str): scan target type moduleList (list): list of modules to run globalOpts (dict): scan options start (bool): start the scan immediately Raises: TypeError: arg type was invalid ValueError: arg value was invalid Todo: Eventually change this to be able to control multiple scan instances """ if not isinstance(globalOpts, dict): raise TypeError( f"globalOpts is {type(globalOpts)}; expected dict()") if not globalOpts: raise ValueError("globalOpts is empty") self.__config = deepcopy(globalOpts) self.__dbh = SpiderFootDb(self.__config) if not isinstance(scanName, str): raise TypeError(f"scanName is {type(scanName)}; expected str()") if not scanName: raise ValueError("scanName value is blank") self.__scanName = scanName if not isinstance(scanId, str): raise TypeError(f"scanId is {type(scanId)}; expected str()") if not scanId: raise ValueError("scanId value is blank") if not isinstance(targetValue, str): raise TypeError( f"targetValue is {type(targetValue)}; expected str()") if not targetValue: raise ValueError("targetValue value is blank") self.__targetValue = targetValue if not isinstance(targetType, str): raise TypeError( f"targetType is {type(targetType)}; expected str()") if not targetType: raise ValueError("targetType value is blank") self.__targetType = targetType if not isinstance(moduleList, list): raise TypeError( f"moduleList is {type(moduleList)}; expected list()") if not moduleList: raise ValueError("moduleList is empty") self.__moduleList = moduleList self.__sf = SpiderFoot(self.__config) self.__sf.dbh = self.__dbh # Create a unique ID for this scan in the back-end DB. if not isinstance(scanId, str): raise TypeError(f"scanId is {type(scanId)}; expected str()") if scanId: self.__scanId = scanId else: self.__scanId = self.__sf.genScanInstanceId() self.__sf.scanId = self.__scanId self.__dbh.scanInstanceCreate(self.__scanId, self.__scanName, self.__targetValue) # Create our target try: self.__target = SpiderFootTarget(self.__targetValue, self.__targetType) except (TypeError, ValueError) as e: self.__sf.status(f"Scan [{self.__scanId}] failed: {e}") self.__setStatus("ERROR-FAILED", None, time.time() * 1000) raise ValueError(f"Invalid target: {e}") # Save the config current set for this scan self.__config['_modulesenabled'] = self.__moduleList self.__dbh.scanConfigSet( self.__scanId, self.__sf.configSerialize(deepcopy(self.__config))) # Process global options that point to other places for data # If a SOCKS server was specified, set it up if self.__config['_socks1type']: socksDns = self.__config['_socks6dns'] socksAddr = self.__config['_socks2addr'] socksPort = int(self.__config['_socks3port']) socksUsername = self.__config['_socks4user'] or '' socksPassword = self.__config['_socks5pwd'] or '' proxy = f"{socksAddr}:{socksPort}" if socksUsername or socksPassword: proxy = "%s:%s@%s" % (socksUsername, socksPassword, proxy) if self.__config['_socks1type'] == '4': proxy = 'socks4://' + proxy elif self.__config['_socks1type'] == '5': proxy = 'socks5://' + proxy elif self.__config['_socks1type'] == 'HTTP': proxy = 'http://' + proxy elif self.__config['_socks1type'] == 'TOR': proxy = 'socks5h://' + proxy else: raise ValueError( f"Invalid SOCKS proxy type: {self.__config['_socks1ttype']}" ) self.__sf.debug( f"SOCKS: {socksAddr}:{socksPort} ({socksUsername}:{socksPassword})" ) self.__sf.socksProxy = proxy else: self.__sf.socksProxy = None # Override the default DNS server if self.__config['_dnsserver']: res = dns.resolver.Resolver() res.nameservers = [self.__config['_dnsserver']] dns.resolver.override_system_resolver(res) else: dns.resolver.restore_system_resolver() # Set the user agent self.__config['_useragent'] = self.__sf.optValueToData( self.__config['_useragent']) # Get internet TLDs tlddata = self.__sf.cacheGet("internet_tlds", self.__config['_internettlds_cache']) # If it wasn't loadable from cache, load it from scratch if tlddata is None: self.__config['_internettlds'] = self.__sf.optValueToData( self.__config['_internettlds']) self.__sf.cachePut("internet_tlds", self.__config['_internettlds']) else: self.__config["_internettlds"] = tlddata.splitlines() self.__setStatus("INITIALIZING", time.time() * 1000, None) if start: self.__startScan()
def startScan(self): global globalScanStatus self.ts = threading.local() self.ts.moduleInstances = dict() self.ts.sf = SpiderFoot(self.temp['config']) self.ts.config = deepcopy(self.temp['config']) self.ts.dbh = SpiderFootDb(self.temp['config']) self.ts.targetValue = self.temp['targetValue'] self.ts.targetType = self.temp['targetType'] self.ts.moduleList = self.temp['moduleList'] self.ts.modconfig = dict() self.ts.scanName = self.temp['scanName'] self.ts.scanId = self.temp['scanId'] aborted = False self.ts.sf.setDbh(self.ts.dbh) # Create a unique ID for this scan and create it in the back-end DB. self.ts.sf.setGUID(self.ts.scanId) self.ts.dbh.scanInstanceCreate(self.ts.scanId, self.ts.scanName, self.ts.targetValue) self.setStatus("STARTING", time.time() * 1000, None) # Create our target target = SpiderFootTarget(self.ts.targetValue, self.ts.targetType) # Save the config current set for this scan self.ts.config['_modulesenabled'] = self.ts.moduleList self.ts.dbh.scanConfigSet( self.ts.scanId, self.ts.sf.configSerialize(deepcopy(self.ts.config))) self.ts.sf.status("Scan [" + self.ts.scanId + "] initiated.") # moduleList = list of modules the user wants to run try: # Process global options that point to other places for data # Save default socket methods that will be overridden if not hasattr(socket, 'savedsocket'): socket.savedsocket = socket.socket socket.savedcreate_connection = socket.create_connection #socket.savedgetaddrinfo = socket.getaddrinfo # If a SOCKS server was specified, set it up if self.ts.config['_socks1type'] != '': socksType = socks.PROXY_TYPE_SOCKS4 socksDns = self.ts.config['_socks6dns'] socksAddr = self.ts.config['_socks2addr'] socksPort = int(self.ts.config['_socks3port']) socksUsername = '' socksPassword = '' if self.ts.config['_socks1type'] == '4': socksType = socks.PROXY_TYPE_SOCKS4 if self.ts.config['_socks1type'] == '5': socksType = socks.PROXY_TYPE_SOCKS5 socksUsername = self.ts.config['_socks4user'] socksPassword = self.ts.config['_socks5pwd'] if self.ts.config['_socks1type'] == 'HTTP': socksType = socks.PROXY_TYPE_HTTP if self.ts.config['_socks1type'] == 'TOR': socksType = socks.PROXY_TYPE_SOCKS5 self.ts.sf.debug("SOCKS: " + socksAddr + ":" + str(socksPort) + \ "(" + socksUsername + ":" + socksPassword + ")") socks.setdefaultproxy(socksType, socksAddr, socksPort, socksDns, socksUsername, socksPassword) # Override the default socket and getaddrinfo calls with the # SOCKS ones. Just ensure we don't also try and SOCKS-proxy # connectivity to the TOR control port. def _create_connection(address, timeout=None, source_address=None): if socksAddr not in address: sock = socks.socksocket() sock.setproxy(socks.PROXY_TYPE_SOCKS5, socksAddr, socksPort) sock.settimeout(self.ts.config['_fetchtimeout']) sock.connect(address) return sock else: sock = socket.socket sock.settimeout(self.ts.config['_fetchtimeout']) sock.connect(address) return sock socket.socket = socks.socksocket socket.setdefaulttimeout(self.ts.config['_fetchtimeout']) socket.create_connection = _create_connection #socket.getaddrinfo = socks.getaddrinfo self.ts.sf.updateSocket(socket) else: # BUG: If the user had a SOCKS proxy set # and then decided to unset it, the original socket class # is not reverted to its default state - we still have # the SOCKS version of socket. socket.socket = socket.savedsocket socket.setdefaulttimeout(self.ts.config['_fetchtimeout']) socket.create_connection = socket.savedcreate_connection #socket.getaddrinfo = socket.savedgetaddrinfo self.ts.sf.revertSocket() # Override the default DNS server if self.ts.config['_dnsserver'] != "": res = dns.resolver.Resolver() res.nameservers = [self.ts.config['_dnsserver']] dns.resolver.override_system_resolver(res) else: dns.resolver.restore_system_resolver() # Set the user agent self.ts.config['_useragent'] = self.ts.sf.optValueToData( self.ts.config['_useragent']) # Get internet TLDs tlddata = self.ts.sf.cacheGet( "internet_tlds", self.ts.config['_internettlds_cache']) # If it wasn't loadable from cache, load it from scratch if tlddata is None: self.ts.config['_internettlds'] = self.ts.sf.optValueToData( self.ts.config['_internettlds']) self.ts.sf.cachePut("internet_tlds", self.ts.config['_internettlds']) else: self.ts.config["_internettlds"] = tlddata.splitlines() for modName in self.ts.moduleList: if modName == '': continue module = __import__('modules.' + modName, globals(), locals(), [modName]) mod = getattr(module, modName)() mod.__name__ = modName # Module may have been renamed or removed if modName not in self.ts.config['__modules__']: continue # Set up the module # Configuration is a combined global config with module-specific options self.ts.modconfig[modName] = deepcopy( self.ts.config['__modules__'][modName]['opts']) for opt in self.ts.config.keys(): self.ts.modconfig[modName][opt] = deepcopy( self.ts.config[opt]) mod.clearListeners( ) # clear any listener relationships from the past mod.setup(self.ts.sf, self.ts.modconfig[modName]) mod.setDbh(self.ts.dbh) mod.setScanId(self.ts.scanId) # Give modules a chance to 'enrich' the original target with # aliases of that target. newTarget = mod.enrichTarget(target) if newTarget is not None: target = newTarget self.ts.moduleInstances[modName] = mod # Override the module's local socket module # to be the SOCKS one. if self.ts.config['_socks1type'] != '': mod._updateSocket(socket) # Set up event output filters if requested if self.ts.config['__outputfilter']: mod.setOutputFilter(self.ts.config['__outputfilter']) self.ts.sf.status(modName + " module loaded.") # Register listener modules and then start all modules sequentially for module in self.ts.moduleInstances.values(): # Register the target with the module module.setTarget(target) for listenerModule in self.ts.moduleInstances.values(): # Careful not to register twice or you will get duplicate events if listenerModule in module._listenerModules: continue # Note the absence of a check for whether a module can register # to itself. That is intentional because some modules will # act on their own notifications (e.g. sfp_dns)! if listenerModule.watchedEvents() is not None: module.registerListener(listenerModule) # Now we are ready to roll.. self.setStatus("RUNNING") # Create a pseudo module for the root event to originate from psMod = SpiderFootPlugin() psMod.__name__ = "SpiderFoot UI" psMod.setTarget(target) psMod.clearListeners() for mod in self.ts.moduleInstances.values(): if mod.watchedEvents() is not None: psMod.registerListener(mod) # Create the "ROOT" event which un-triggered modules will link events to rootEvent = SpiderFootEvent("ROOT", self.ts.targetValue, "", None) psMod.notifyListeners(rootEvent) firstEvent = SpiderFootEvent(self.ts.targetType, self.ts.targetValue, "SpiderFoot UI", rootEvent) psMod.notifyListeners(firstEvent) # If in interactive mode, loop through this shared global variable # waiting for inputs, and process them until my status is set to # FINISHED. # Check in case the user requested to stop the scan between modules # initializing for module in self.ts.moduleInstances.values(): if module.checkForStop(): self.setStatus('ABORTING') aborted = True break if aborted: self.ts.sf.status("Scan [" + self.ts.scanId + "] aborted.") self.setStatus("ABORTED", None, time.time() * 1000) else: self.ts.sf.status("Scan [" + self.ts.scanId + "] completed.") self.setStatus("FINISHED", None, time.time() * 1000) except BaseException as e: exc_type, exc_value, exc_traceback = sys.exc_info() self.ts.sf.error("Unhandled exception (" + e.__class__.__name__ + ") " + \ "encountered during scan. Please report this as a bug: " + \ repr(traceback.format_exception(exc_type, exc_value, exc_traceback)), False) self.ts.sf.status("Scan [" + self.ts.scanId + "] failed: " + str(e)) self.setStatus("ERROR-FAILED", None, time.time() * 1000) self.ts.dbh.close() del self.ts del self.temp
def __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts, start=True): """Initialize SpiderFootScanner object. Args: scanName (str): name of the scan scanId (str): unique ID of the scan scanTarget (str): scan target targetType (str): scan target type moduleList (list): list of modules to run globalOpts (dict): scan options start (bool): start the scan immediately Returns: None Todo: Eventually change this to be able to control multiple scan instances """ if not isinstance(scanName, str): raise TypeError("scanName is %s; expected str()" % type(scanName)) if not isinstance(scanId, str): raise TypeError("scanId is %s; expected str()" % type(scanId)) if not isinstance(scanTarget, str): raise TypeError("scanTarget is %s; expected str()" % type(scanTarget)) if not isinstance(targetType, str): raise TypeError("targetType is %s; expected str()" % type(targetType)) if not isinstance(moduleList, list): raise TypeError("moduleList is %s; expected list()" % type(moduleList)) if not isinstance(globalOpts, dict): raise TypeError("globalOpts is %s; expected dict()" % type(globalOpts)) if not globalOpts: raise ValueError("globalOpts is empty") self.moduleInstances = dict() self.config = deepcopy(globalOpts) self.sf = SpiderFoot(self.config) self.dbh = SpiderFootDb(self.config) self.targetValue = scanTarget self.targetType = targetType self.moduleList = moduleList self.modconfig = dict() self.scanName = scanName self.sf.setDbh(self.dbh) # Create a unique ID for this scan and create it in the back-end DB. self.scanId = scanId self.sf.setGUID(self.scanId) self.dbh.scanInstanceCreate(self.scanId, self.scanName, self.targetValue) # Create our target try: self.target = SpiderFootTarget(self.targetValue, self.targetType) except ValueError as e: self.sf.status("Scan [%s] failed: %s" % (self.scanId, e)) self.setStatus("ERROR-FAILED", None, time.time() * 1000) raise ValueError("Invalid target: %s" % e) except TypeError as e: self.sf.status("Scan [%s] failed: %s" % (self.scanId, e)) self.setStatus("ERROR-FAILED", None, time.time() * 1000) raise TypeError("Invalid target: %s" % e) # Save the config current set for this scan self.config['_modulesenabled'] = self.moduleList self.dbh.scanConfigSet(self.scanId, self.sf.configSerialize(deepcopy(self.config))) # Process global options that point to other places for data # If a SOCKS server was specified, set it up if self.config['_socks1type']: socksDns = self.config['_socks6dns'] socksAddr = self.config['_socks2addr'] socksPort = int(self.config['_socks3port']) socksUsername = self.config['_socks4user'] or '' socksPassword = self.config['_socks5pwd'] or '' proxy = "%s:%s" % (socksAddr, socksPort) if socksUsername or socksPassword: proxy = "%s:%s@%s" % (socksUsername, socksPassword, proxy) if self.config['_socks1type'] == '4': proxy = 'socks4://' + proxy elif self.config['_socks1type'] == '5': proxy = 'socks5://' + proxy elif self.config['_socks1type'] == 'HTTP': proxy = 'http://' + proxy elif self.config['_socks1type'] == 'TOR': proxy = 'socks5h://' + proxy else: raise ValueError("Invalid SOCKS proxy type: %s" % self.config["_socks1ttype"]) self.sf.debug("SOCKS: %s:%s (%s:%s)" % (socksAddr, socksPort, socksUsername, socksPassword)) self.sf.updateSocket(proxy) else: self.sf.revertSocket() # Override the default DNS server if self.config['_dnsserver']: res = dns.resolver.Resolver() res.nameservers = [self.config['_dnsserver']] dns.resolver.override_system_resolver(res) else: dns.resolver.restore_system_resolver() # Set the user agent self.config['_useragent'] = self.sf.optValueToData(self.config['_useragent']) # Get internet TLDs tlddata = self.sf.cacheGet("internet_tlds", self.config['_internettlds_cache']) # If it wasn't loadable from cache, load it from scratch if tlddata is None: self.config['_internettlds'] = self.sf.optValueToData(self.config['_internettlds']) self.sf.cachePut("internet_tlds", self.config['_internettlds']) else: self.config["_internettlds"] = tlddata.splitlines() self.setStatus("INITIALIZING", time.time() * 1000, None) if start: self.startScan()
def startScan(self): """Start running a scan.""" self.moduleInstances = dict() self.sf = SpiderFoot(self.temp['config']) self.config = deepcopy(self.temp['config']) self.dbh = SpiderFootDb(self.temp['config']) self.targetValue = self.temp['targetValue'] self.targetType = self.temp['targetType'] self.moduleList = self.temp['moduleList'] self.modconfig = dict() self.scanName = self.temp['scanName'] self.scanId = self.temp['scanId'] aborted = False self.sf.setDbh(self.dbh) # Create a unique ID for this scan and create it in the back-end DB. self.sf.setGUID(self.scanId) self.dbh.scanInstanceCreate(self.scanId, self.scanName, self.targetValue) self.setStatus("STARTING", time.time() * 1000, None) # Create our target try: target = SpiderFootTarget(self.targetValue, self.targetType) except BaseException as e: self.sf.status("Scan [%s] failed: %s" % (self.scanId, e)) self.setStatus("ERROR-FAILED", None, time.time() * 1000) return None # Save the config current set for this scan self.config['_modulesenabled'] = self.moduleList self.dbh.scanConfigSet(self.scanId, self.sf.configSerialize(deepcopy(self.config))) self.sf.status("Scan [" + self.scanId + "] initiated.") # moduleList = list of modules the user wants to run try: # Process global options that point to other places for data # If a SOCKS server was specified, set it up if self.config['_socks1type'] != '': socksDns = self.config['_socks6dns'] socksAddr = self.config['_socks2addr'] socksPort = int(self.config['_socks3port']) socksUsername = self.config['_socks4user'] or '' socksPassword = self.config['_socks5pwd'] or '' creds = "" if socksUsername and socksPassword: creds = socksUsername + ":" + socksPassword + "@" proxy = creds + socksAddr + ":" + str(socksPort) if self.config['_socks1type'] == '4': proxy = 'socks4://' + proxy elif self.config['_socks1type'] == '5': proxy = 'socks5://' + proxy elif self.config['_socks1type'] == 'HTTP': proxy = 'http://' + proxy elif self.config['_socks1type'] == 'TOR': proxy = 'socks5h://' + proxy self.sf.debug("SOCKS: " + socksAddr + ":" + str(socksPort) + \ "(" + socksUsername + ":" + socksPassword + ")") self.sf.updateSocket(proxy) else: self.sf.revertSocket() # Override the default DNS server if self.config['_dnsserver'] != "": res = dns.resolver.Resolver() res.nameservers = [self.config['_dnsserver']] dns.resolver.override_system_resolver(res) else: dns.resolver.restore_system_resolver() # Set the user agent self.config['_useragent'] = self.sf.optValueToData( self.config['_useragent']) # Get internet TLDs tlddata = self.sf.cacheGet("internet_tlds", self.config['_internettlds_cache']) # If it wasn't loadable from cache, load it from scratch if tlddata is None: self.config['_internettlds'] = self.sf.optValueToData( self.config['_internettlds']) self.sf.cachePut("internet_tlds", self.config['_internettlds']) else: self.config["_internettlds"] = tlddata.splitlines() for modName in self.moduleList: if modName == '': continue try: module = __import__('modules.' + modName, globals(), locals(), [modName]) except ImportError: self.sf.error("Failed to load module: " + modName, False) continue mod = getattr(module, modName)() mod.__name__ = modName # Module may have been renamed or removed if modName not in self.config['__modules__']: continue # Set up the module # Configuration is a combined global config with module-specific options self.modconfig[modName] = deepcopy(self.config['__modules__'][modName]['opts']) for opt in list(self.config.keys()): self.modconfig[modName][opt] = deepcopy(self.config[opt]) mod.clearListeners() # clear any listener relationships from the past mod.setup(self.sf, self.modconfig[modName]) mod.setDbh(self.dbh) mod.setScanId(self.scanId) # Give modules a chance to 'enrich' the original target with # aliases of that target. newTarget = mod.enrichTarget(target) if newTarget is not None: target = newTarget self.moduleInstances[modName] = mod # Override the module's local socket module # to be the SOCKS one. if self.config['_socks1type'] != '': mod._updateSocket(socket) # Set up event output filters if requested if self.config['__outputfilter']: mod.setOutputFilter(self.config['__outputfilter']) self.sf.status(modName + " module loaded.") # Register listener modules and then start all modules sequentially for module in list(self.moduleInstances.values()): # Register the target with the module module.setTarget(target) for listenerModule in list(self.moduleInstances.values()): # Careful not to register twice or you will get duplicate events if listenerModule in module._listenerModules: continue # Note the absence of a check for whether a module can register # to itself. That is intentional because some modules will # act on their own notifications (e.g. sfp_dns)! if listenerModule.watchedEvents() is not None: module.registerListener(listenerModule) # Now we are ready to roll.. self.setStatus("RUNNING") # Create a pseudo module for the root event to originate from psMod = SpiderFootPlugin() psMod.__name__ = "SpiderFoot UI" psMod.setTarget(target) psMod.setDbh(self.dbh) psMod.clearListeners() for mod in list(self.moduleInstances.values()): if mod.watchedEvents() is not None: psMod.registerListener(mod) # Create the "ROOT" event which un-triggered modules will link events to rootEvent = SpiderFootEvent("ROOT", self.targetValue, "", None) psMod.notifyListeners(rootEvent) firstEvent = SpiderFootEvent(self.targetType, self.targetValue, "SpiderFoot UI", rootEvent) psMod.notifyListeners(firstEvent) # Special case.. check if an INTERNET_NAME is also a domain if self.targetType == 'INTERNET_NAME': if self.sf.isDomain(self.targetValue, self.config['_internettlds']): firstEvent = SpiderFootEvent('DOMAIN_NAME', self.targetValue, "SpiderFoot UI", rootEvent) psMod.notifyListeners(firstEvent) # If in interactive mode, loop through this shared global variable # waiting for inputs, and process them until my status is set to # FINISHED. # Check in case the user requested to stop the scan between modules # initializing for module in list(self.moduleInstances.values()): if module.checkForStop(): self.setStatus('ABORTING') aborted = True break if aborted: self.sf.status("Scan [" + self.scanId + "] aborted.") self.setStatus("ABORTED", None, time.time() * 1000) else: self.sf.status("Scan [" + self.scanId + "] completed.") self.setStatus("FINISHED", None, time.time() * 1000) except BaseException as e: exc_type, exc_value, exc_traceback = sys.exc_info() self.sf.error("Unhandled exception (" + e.__class__.__name__ + ") " + \ "encountered during scan. Please report this as a bug: " + \ repr(traceback.format_exception(exc_type, exc_value, exc_traceback)), False) self.sf.status("Scan [" + self.scanId + "] failed: " + str(e)) self.setStatus("ERROR-FAILED", None, time.time() * 1000) self.dbh.close()
def test_init(self): """ Test __init__ """ #target = SpiderFootTarget('junk', 'junk') #self.assertEqual(target, None) target_value = '127.0.0.1' target_type = 'IP_ADDRESS' target = SpiderFootTarget(target_value, target_type) self.assertEqual(target.getType(), target_type) self.assertEqual(target.getValue(), target_value) target_value = '::1' target_type = 'IPV6_ADDRESS' target = SpiderFootTarget(target_value, target_type) self.assertEqual(target.getType(), target_type) self.assertEqual(target.getValue(), target_value) target_value = '0.0.0.0/0' target_type = 'NETBLOCK_OWNER' target = SpiderFootTarget(target_value, target_type) self.assertEqual(target.getType(), target_type) self.assertEqual(target.getValue(), target_value) target_value = 'localhost.local' target_type = 'INTERNET_NAME' target = SpiderFootTarget(target_value, target_type) self.assertEqual(target.getType(), target_type) self.assertEqual(target.getValue(), target_value) target_value = '*****@*****.**' target_type = 'EMAILADDR' target = SpiderFootTarget(target_value, target_type) self.assertEqual(target.getType(), target_type) self.assertEqual(target.getValue(), target_value) target_value = 'hello' target_type = 'HUMAN_NAME' target = SpiderFootTarget(target_value, target_type) self.assertEqual(target.getType(), target_type) self.assertEqual(target.getValue(), target_value) target_value = '1234' target_type = 'BGP_AS_OWNER' target = SpiderFootTarget(target_value, target_type) self.assertEqual(target.getType(), target_type) self.assertEqual(target.getValue(), target_value) target_value = '+12345678901' target_type = 'PHONE_NUMBER' target = SpiderFootTarget(target_value, target_type) self.assertEqual(target.getType(), target_type) self.assertEqual(target.getValue(), target_value)