def __init__(self, config):
self.defaultConfig = deepcopy(config)
dbh = SpiderFootDb(self.defaultConfig)
# 'config' supplied will be the defaults, let's supplement them
# now with any configuration which may have previously been
# saved.
sf = SpiderFoot(self.defaultConfig)
self.config = sf.configUnserialize(dbh.configGet(), self.defaultConfig)
if self.config['__webaddr'] == "0.0.0.0":
addr = "<IP of this host>"
else:
addr = self.config['__webaddr']
self.docroot = self.config['__docroot'].rstrip('/')
print ""
print ""
print "*************************************************************"
print " Use SpiderFoot by starting your web browser of choice and "
print " browse to http://" + addr + ":" + str(self.config['__webport']) + self.docroot
print "*************************************************************"
print ""
print ""
def test_handleEvent_no_tool_path_configured_should_set_errorState(self):
"""
Test handleEvent(self, event)
"""
sf = SpiderFoot(self.default_options)
module = sfp_tool_snallygaster()
module.setup(sf, dict())
target_value = 'example target value'
target_type = 'IP_ADDRESS'
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
event_type = 'ROOT'
event_data = 'example data'
event_module = ''
source_event = ''
evt = SpiderFootEvent(event_type, event_data, event_module, source_event)
result = module.handleEvent(evt)
self.assertIsNone(result)
self.assertTrue(module.errorState)
def optsexport(self, pattern):
"""Export configuration
Args:
pattern: TBD
"""
sf = SpiderFoot(self.config)
conf = sf.configSerialize(self.config)
content = ""
for opt in sorted(conf):
if ":_" in opt or opt.startswith("_"):
continue
if pattern:
if pattern in opt:
content += "%s=%s\n" % (opt, conf[opt])
else:
content += "%s=%s\n" % (opt, conf[opt])
cherrypy.response.headers['Content-Disposition'] = 'attachment; filename="SpiderFoot.cfg"'
cherrypy.response.headers['Content-Type'] = "text/plain"
return content
def scanopts(self, id):
ret = dict()
dbh = SpiderFootDb(self.config)
ret['config'] = dbh.scanConfigGet(id)
ret['configdesc'] = dict()
for key in ret['config'].keys():
if ':' not in key:
ret['configdesc'][key] = self.config['__globaloptdescs__'][key]
else:
[modName, modOpt] = key.split(':')
if modName not in self.config['__modules__'].keys():
continue
if modOpt not in self.config['__modules__'][modName][
'optdescs'].keys():
continue
ret['configdesc'][key] = self.config['__modules__'][modName][
'optdescs'][modOpt]
sf = SpiderFoot(self.config)
meta = dbh.scanInstanceGet(id)
if meta[3] != 0:
started = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(meta[3]))
else:
started = "Not yet"
if meta[4] != 0:
finished = time.strftime("%Y-%m-%d %H:%M:%S",
time.localtime(meta[4]))
else:
finished = "Not yet"
ret['meta'] = [meta[0], meta[1], meta[2], started, finished, meta[5]]
return json.dumps(ret)
def clonescan(self, id):
"""
Clone an existing scan (pre-selected options in the newscan page)
Args:
id (str): scan ID to clone
"""
sf = SpiderFoot(self.config)
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
info = dbh.scanInstanceGet(id)
if not info:
return self.error("Invalid scan ID.")
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if scanname == "" or scantarget == "" or len(scanconfig) == 0:
return self.error("Something went wrong internally.")
targetType = sf.targetType(scantarget)
if targetType is None:
# It must be a name, so wrap quotes around it
scantarget = """ + scantarget + """
modlist = scanconfig['_modulesenabled'].split(',')
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(pageid='NEWSCAN', types=types, docroot=self.docroot,
modules=self.config['__modules__'], selectedmods=modlist,
scanname=str(scanname),
scantarget=str(scantarget))
def test_handleEvent_event_data_affiliate_internet_name_not_matching_ad_server_should_not_return_event(self):
sf = SpiderFoot(self.default_options)
module = sfp_stevenblack_hosts()
module.setup(sf, dict())
target_value = 'spiderfoot.net'
target_type = 'INTERNET_NAME'
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
module.opts['_fetchtimeout'] = 15
module.optdescs['_fetchtimeout'] = ''
module.opts['_useragent'] = ''
module.optdescs['_useragent'] = ''
def new_notifyListeners(self, event):
raise Exception(f"Raised event {event.eventType}: {event.data}")
module.notifyListeners = new_notifyListeners.__get__(module, sfp_stevenblack_hosts)
event_type = 'ROOT'
event_data = 'example data'
event_module = ''
source_event = ''
evt = SpiderFootEvent(event_type, event_data, event_module, source_event)
event_type = 'AFFILIATE_INTERNET_NAME'
event_data = 'no.ads.safe.local'
event_module = 'example module'
source_event = evt
evt = SpiderFootEvent(event_type, event_data, event_module, source_event)
result = module.handleEvent(evt)
self.assertIsNone(result)
def test_handleEvent_domain_whois_event_data_not_containing_webframework_string_should_not_create_event(
self):
sf = SpiderFoot(self.default_options)
module = sfp_webframework()
module.setup(sf, dict())
target_value = 'spiderfoot.net'
target_type = 'INTERNET_NAME'
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
def new_notifyListeners(self, event):
raise Exception(f"Raised event {event.eventType}: {event.data}")
module.notifyListeners = new_notifyListeners.__get__(
module, sfp_webframework)
event_type = 'ROOT'
event_data = 'example data'
event_module = ''
source_event = ''
evt = SpiderFootEvent(event_type, event_data, event_module,
source_event)
event_type = 'TARGET_WEB_CONTENT'
event_data = 'example data'
event_module = 'example module'
source_event = evt
evt = SpiderFootEvent(event_type, event_data, event_module,
source_event)
evt.actualSource = "https://spiderfoot.net/"
result = module.handleEvent(evt)
self.assertIsNone(result)
def test_handleEvent(self):
"""
Test handleEvent(self, event)
"""
sf = SpiderFoot(self.default_options)
module = sfp_ipregistry()
module.setup(sf, dict())
target_value = "example target value"
target_type = "IP_ADDRESS"
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
event_type = "ROOT"
event_data = "example data"
event_module = ""
source_event = ""
evt = SpiderFootEvent(event_type, event_data, event_module,
source_event)
result = module.handleEvent(evt)
self.assertIsNone(result)
def test_handleEvent(self):
"""
Test handleEvent(self, event)
"""
sf = SpiderFoot(self.default_options)
module = sfp_networksdb()
module.setup(sf, dict())
target_value = 'example target value'
target_type = 'IP_ADDRESS'
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
event_type = 'ROOT'
event_data = 'example data'
event_module = ''
source_event = ''
evt = SpiderFootEvent(event_type, event_data, event_module,
source_event)
result = module.handleEvent(evt)
self.assertIsNone(result)
def test_handleEvent_no_api_key_should_set_errorState(self):
"""
Test handleEvent(self, event)
"""
sf = SpiderFoot(self.default_options)
module = sfp_certspotter()
module.setup(sf, dict())
target_value = 'example target value'
target_type = 'INTERNET_NAME'
target = SpiderFootTarget(target_value, target_type)
module.setTarget(target)
event_type = 'ROOT'
event_data = 'example data'
event_module = ''
source_event = ''
evt = SpiderFootEvent(event_type, event_data, event_module, source_event)
result = module.handleEvent(evt)
self.assertIsNone(result)
self.assertTrue(module.errorState)
def test_target_type(self):
"""
Test targetType(self, target)
"""
sf = SpiderFoot(dict())
target_type = sf.targetType("0.0.0.0")
self.assertEqual('IP_ADDRESS', target_type)
target_type = sf.targetType("*****@*****.**")
self.assertEqual('EMAILADDR', target_type)
target_type = sf.targetType("0.0.0.0/0")
self.assertEqual('NETBLOCK_OWNER', target_type)
target_type = sf.targetType("+1234567890")
self.assertEqual('PHONE_NUMBER', target_type)
target_type = sf.targetType('"Human Name"')
self.assertEqual('HUMAN_NAME', target_type)
target_type = sf.targetType('"abc123"')
self.assertEqual('USERNAME', target_type)
target_type = sf.targetType("1234567890")
self.assertEqual('BGP_AS_OWNER', target_type)
target_type = sf.targetType("::1")
self.assertEqual('IPV6_ADDRESS', target_type)
target_type = sf.targetType("spiderfoot.net")
self.assertEqual('INTERNET_NAME', target_type)
def test_init(self):
"""
Test __init__(self, options, handle=None):
"""
sf = SpiderFoot(self.default_options)
self.assertIsInstance(sf, SpiderFoot)
def test_init_no_options(self):
"""
Test __init__(self, options, handle=None):
"""
sf = SpiderFoot(dict())
self.assertIsInstance(sf, SpiderFoot)
def test_parse_iban_numbers_should_return_a_list(self):
"""
Test parseIBANNumbers(self, data)
"""
sf = SpiderFoot(self.default_options)
invalid_types = [None, "", list(), dict()]
for invalid_type in invalid_types:
with self.subTest(invalid_type=invalid_type):
ibans = sf.parseIBANNumbers(invalid_type)
self.assertIsInstance(ibans, list)
# Example IBANS from https://www.iban.com/structure
ibans = [
"AL35202111090000000001234567",
"AD1400080001001234567890",
"AT483200000012345864",
"AZ96AZEJ00000000001234567890",
"BH02CITI00001077181611",
"BY86AKBB10100000002966000000",
"BE71096123456769",
"BA393385804800211234",
"BR1500000000000010932840814P2",
"BG18RZBB91550123456789",
"CR23015108410026012345",
"HR1723600001101234565",
"CY21002001950000357001234567",
"CZ5508000000001234567899",
"DK9520000123456789",
"DO22ACAU00000000000123456789",
"EG800002000156789012345180002",
"SV43ACAT00000000000000123123",
"EE471000001020145685",
"FO9264600123456789",
"FI1410093000123458",
"FR7630006000011234567890189",
"GE60NB0000000123456789",
"DE75512108001245126199",
"GI04BARC000001234567890",
"GR9608100010000001234567890",
"GL8964710123456789",
"GT20AGRO00000000001234567890",
"VA59001123000012345678",
"HU93116000060000000012345676",
"IS750001121234563108962099",
"IQ20CBIQ861800101010500",
"IE64IRCE92050112345678",
"IL170108000000012612345",
"IT60X0542811101000000123456",
"JO71CBJO0000000000001234567890",
"KZ563190000012344567",
"XK051212012345678906",
"KW81CBKU0000000000001234560101",
"LV97HABA0012345678910",
"LB92000700000000123123456123",
"LI7408806123456789012",
"LT601010012345678901",
"LU120010001234567891",
"MT31MALT01100000000000000000123",
"MR1300020001010000123456753",
"MU43BOMM0101123456789101000MUR",
"MD21EX000000000001234567",
"MC5810096180790123456789085",
"ME25505000012345678951",
"NL02ABNA0123456789",
"MK07200002785123453",
"NO8330001234567",
"PK36SCBL0000001123456702",
"PS92PALS000000000400123456702",
"PL10105000997603123456789123",
"PT50002700000001234567833",
"QA54QNBA000000000000693123456",
"RO09BCYP0000001234567890",
"LC14BOSL123456789012345678901234",
"SM76P0854009812123456789123",
"ST23000200000289355710148",
"SA4420000001234567891234",
"RS35105008123123123173",
"SC52BAHL01031234567890123456USD",
"SK8975000000000012345671",
"SI56192001234567892",
"ES7921000813610123456789",
"SE7280000810340009783242",
"CH5604835012345678009",
"TL380010012345678910106",
"TN5904018104004942712345",
"TR320010009999901234567890",
"UA903052992990004149123456789",
"AE460090000000123456789",
"GB33BUKB20201555555555",
"VG21PACG0000000123456789"
]
for iban in ibans:
with self.subTest(iban=iban):
parse_ibans = sf.parseIBANNumbers(iban)
self.assertIsInstance(parse_ibans, list)
self.assertIn(iban, parse_ibans)
# Invalid IBANs
ibans = [
# Invalid country code
"ZZ21PACG0000000123456789",
# Invalid length for country code
"VG123456789012345",
# Invalid mod 97 remainder
"VG21PACG0000000123456111"
]
for iban in ibans:
with self.subTest(iban=iban):
parse_ibans = sf.parseIBANNumbers(iban)
self.assertIsInstance(parse_ibans, list)
self.assertNotIn(iban, parse_ibans)
def test_setup(self):
sf = SpiderFoot(self.default_options)
module = sfp_talosintel()
module.setup(sf, dict())
def test_setup(self):
sf = SpiderFoot(self.default_options)
module = sfp_torexits()
module.setup(sf, dict())
def test_init(self):
"""
Test __init__(self, options, handle=None):
"""
sf = SpiderFoot(self.default_options)
self.assertEqual('TBD', 'TBD')
'_socks2addr': 'SOCKS Server IP Address.',
'_socks3port': 'SOCKS Server TCP Port. Usually 1080 for 4/5, 8080 for HTTP and 9050 for TOR.',
'_socks4user': '******',
'_socks5pwd': "SOCKS Password. Valid only for SOCKS5 servers.",
'_socks6dns': "Resolve DNS through the SOCKS proxy? Has no affect when TOR is used: Will always be True.",
'_torctlport': "The port TOR is taking control commands on. This is necessary for SpiderFoot to tell TOR to re-circuit when it suspects anonymity is compromised.",
'_modulesenabled': "Modules enabled for the scan." # This is a hack to get a description for an option not actually available.
}
if __name__ == '__main__':
if len(sys.argv) > 1:
(addr, port) = sys.argv[1].split(":")
sfConfig['__webaddr'] = addr
sfConfig['__webport'] = int(port)
sf = SpiderFoot(sfConfig)
sfModules = dict()
# Go through each module in the modules directory with a .py extension
for filename in os.listdir(sf.myPath() + '/modules/'):
if filename.startswith("sfp_") and filename.endswith(".py"):
# Skip the module template and debugging modules
if filename == "sfp_template.py" or filename == 'sfp_stor_print.py':
continue
modName = filename.split('.')[0]
# Load and instantiate the module
sfModules[modName] = dict()
mod = __import__('modules.' + modName, globals(), locals(), [modName])
sfModules[modName]['object'] = getattr(mod, modName)()
sfModules[modName]['name'] = sfModules[modName]['object'].__doc__.split(":", 5)[0]
def start_web_server(sfWebUiConfig, sfConfig):
"""Start the web server so you can start looking at results
Args:
sfWebUiConfig (dict): web server options
sfConfig (dict): SpiderFoot config options
"""
web_host = sfWebUiConfig.get('host', '127.0.0.1')
web_port = sfWebUiConfig.get('port', 5001)
web_root = sfWebUiConfig.get('root', '/')
cherrypy.config.update({
'log.screen': False,
'server.socket_host': web_host,
'server.socket_port': int(web_port)
})
log.info(f"Starting web server at {web_host}:{web_port} ...")
# Disable auto-reloading of content
cherrypy.engine.autoreload.unsubscribe()
sf = SpiderFoot(sfConfig)
# Enable access to static files via the web directory
conf = {
'/query': {
'tools.encode.text_only': False,
'tools.encode.add_charset': True,
},
'/static': {
'tools.staticdir.on': True,
'tools.staticdir.dir': 'static',
'tools.staticdir.root': sf.myPath()
}
}
passwd_file = sf.dataPath() + '/passwd'
if os.path.isfile(passwd_file):
if not os.access(passwd_file, os.R_OK):
log.error("Could not read passwd file. Permission denied.")
sys.exit(-1)
secrets = dict()
pw = open(passwd_file, 'r')
for line in pw.readlines():
if ':' not in line:
log.error(
"Incorrect format of passwd file, must be username:password on each line."
)
sys.exit(-1)
u = line.strip().split(":")[0]
p = ':'.join(line.strip().split(":")[1:])
if not u or not p:
log.error(
"Incorrect format of passwd file, must be username:password on each line."
)
sys.exit(-1)
secrets[u] = p
if secrets:
log.info("Enabling authentication based on supplied passwd file.")
conf['/'] = {
'tools.auth_digest.on':
True,
'tools.auth_digest.realm':
web_host,
'tools.auth_digest.get_ha1':
auth_digest.get_ha1_dict_plain(secrets),
'tools.auth_digest.key':
random.SystemRandom().randint(0, 99999999)
}
else:
warn_msg = "\n********************************************************************\n"
warn_msg += "Warning: passwd file contains no passwords. Authentication disabled.\n"
warn_msg += "********************************************************************\n"
log.warning(warn_msg)
else:
warn_msg = "\n********************************************************************\n"
warn_msg += "Please consider adding authentication to protect this instance!\n"
warn_msg += "Refer to https://www.spiderfoot.net/documentation/#security.\n"
warn_msg += "********************************************************************\n"
log.warning(warn_msg)
if web_host == "0.0.0.0": # nosec
url = f"http://<IP of this host>:{web_port}{web_root}"
else:
url = f"http://{web_host}:{web_port}{web_root}"
key_path = sf.dataPath() + '/spiderfoot.key'
crt_path = sf.dataPath() + '/spiderfoot.crt'
if os.path.isfile(key_path) and os.path.isfile(crt_path):
if not os.access(crt_path, os.R_OK):
log.critical(f"Could not read {crt_path} file. Permission denied.")
sys.exit(-1)
if not os.access(key_path, os.R_OK):
log.critical(f"Could not read {key_path} file. Permission denied.")
sys.exit(-1)
log.info("Enabling SSL based on supplied key and certificate file.")
cherrypy.server.ssl_module = 'builtin'
cherrypy.server.ssl_certificate = crt_path
cherrypy.server.ssl_private_key = key_path
url = url.replace("http://", "https://")
print("")
print("*************************************************************")
print(" Use SpiderFoot by starting your web browser of choice and ")
print(f" browse to {url}")
print("*************************************************************")
print("")
cherrypy.quickstart(SpiderFootWebUi(sfWebUiConfig, sfConfig),
script_name=web_root,
config=conf)
def startscan(self, scanname, scantarget, modulelist, typelist, usecase):
global globalScanStatus
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
types = dbh.eventTypes()
targetType = None
[scanname, scantarget] = self.cleanUserInput([scanname, scantarget])
if scanname == "" or scantarget == "":
return self.error("Form incomplete.")
if typelist == "" and modulelist == "" and usecase == "":
return self.error("Form incomplete.")
# User selected modules
if modulelist != "":
modlist = modulelist.replace('module_', '').split(',')
# User selected types
if len(modlist) == 0 and typelist != "":
typesx = typelist.replace('type_', '').split(',')
# 1. Find all modules that produce the requested types
modlist = sf.modulesProducing(typesx)
newmods = deepcopy(modlist)
newmodcpy = deepcopy(newmods)
# 2. For each type those modules consume, get modules producing
while len(newmodcpy) > 0:
for etype in sf.eventsToModules(newmodcpy):
xmods = sf.modulesProducing([etype])
for mod in xmods:
if mod not in modlist:
modlist.append(mod)
newmods.append(mod)
newmodcpy = deepcopy(newmods)
newmods = list()
# User selected a use case
if len(modlist) == 0 and usecase != "":
for mod in self.config['__modules__']:
if usecase == 'all' or usecase in self.config['__modules__'][
mod]['cats']:
modlist.append(mod)
# Add our mandatory storage module..
if "sfp__stor_db" not in modlist:
modlist.append("sfp__stor_db")
modlist.sort()
targetType = sf.targetType(scantarget)
if targetType is None:
return self.error("Invalid target type. Could not recognize it as " + \
"an IP address, IP subnet, domain name or host name.")
# Start running a new scan
scanId = sf.genScanInstanceGUID(scanname)
t = SpiderFootScanner(scanname, scantarget.lower(), targetType, scanId,
modlist, cfg, modopts)
t.start()
# Wait until the scan has initialized
while globalScanStatus.getStatus(scanId) is None:
print "[info] Waiting for the scan to initialize..."
time.sleep(1)
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
return templ.render(id=scanId,
name=scanname,
docroot=self.docroot,
status=globalScanStatus.getStatus(scanId),
pageid="SCANLIST")
def start_scan(sfConfig, sfModules, args):
global dbh
global scanId
dbh = SpiderFootDb(sfConfig, init=True)
sf = SpiderFoot(sfConfig)
if args.modules:
log.info("Modules available:")
for m in sorted(sfModules.keys()):
if "__" in m:
continue
print(('{0:25} {1}'.format(m, sfModules[m]['descr'])))
sys.exit(0)
if args.types:
log.info("Types available:")
typedata = dbh.eventTypes()
types = dict()
for r in typedata:
types[r[1]] = r[0]
for t in sorted(types.keys()):
print(('{0:45} {1}'.format(t, types[t])))
sys.exit(0)
if not args.s:
log.error(
"You must specify a target when running in scan mode. Try --help for guidance."
)
sys.exit(-1)
if args.x and not args.t:
log.error("-x can only be used with -t. Use --help for guidance.")
sys.exit(-1)
if args.x and args.m:
log.error(
"-x can only be used with -t and not with -m. Use --help for guidance."
)
sys.exit(-1)
if args.r and (args.o and args.o not in ["tab", "csv"]):
log.error("-r can only be used when your output format is tab or csv.")
sys.exit(-1)
if args.H and (args.o and args.o not in ["tab", "csv"]):
log.error("-H can only be used when your output format is tab or csv.")
sys.exit(-1)
if args.D and args.o != "csv":
log.error("-D can only be used when using the csv output format.")
sys.exit(-1)
target = args.s
# Usernames and names - quoted on the commandline - won't have quotes,
# so add them.
if " " in target:
target = f"\"{target}\""
if "." not in target and not target.startswith("+") and '"' not in target:
target = f"\"{target}\""
targetType = sf.targetType(target)
if not targetType:
log.error(f"Could not determine target type. Invalid target: {target}")
sys.exit(-1)
target = target.strip('"')
modlist = list()
if not args.t and not args.m:
log.warning(
"You didn't specify any modules or types, so all will be enabled.")
for m in list(sfModules.keys()):
if "__" in m:
continue
modlist.append(m)
signal.signal(signal.SIGINT, handle_abort)
# If the user is scanning by type..
# 1. Find modules producing that type
if args.t:
types = args.t
modlist = sf.modulesProducing(types)
newmods = deepcopy(modlist)
newmodcpy = deepcopy(newmods)
# 2. For each type those modules consume, get modules producing
while len(newmodcpy) > 0:
for etype in sf.eventsToModules(newmodcpy):
xmods = sf.modulesProducing([etype])
for mod in xmods:
if mod not in modlist:
modlist.append(mod)
newmods.append(mod)
newmodcpy = deepcopy(newmods)
newmods = list()
# Easier if scanning by module
if args.m:
modlist = list(filter(None, args.m.split(",")))
# Add sfp__stor_stdout to the module list
typedata = dbh.eventTypes()
types = dict()
for r in typedata:
types[r[1]] = r[0]
sfp__stor_stdout_opts = sfConfig['__modules__']['sfp__stor_stdout']['opts']
sfp__stor_stdout_opts['_eventtypes'] = types
if args.f:
if args.f and not args.t:
log.error("You can only use -f with -t. Use --help for guidance.")
sys.exit(-1)
sfp__stor_stdout_opts['_showonlyrequested'] = True
if args.F:
sfp__stor_stdout_opts['_requested'] = args.F.split(",")
sfp__stor_stdout_opts['_showonlyrequested'] = True
if args.o:
sfp__stor_stdout_opts['_format'] = args.o
if args.t:
sfp__stor_stdout_opts['_requested'] = args.t.split(",")
if args.n:
sfp__stor_stdout_opts['_stripnewline'] = True
if args.r:
sfp__stor_stdout_opts['_showsource'] = True
if args.S:
sfp__stor_stdout_opts['_maxlength'] = args.S
if args.D:
sfp__stor_stdout_opts['_csvdelim'] = args.D
if args.x:
tmodlist = list()
modlist = list()
xmods = sf.modulesConsuming([targetType])
for mod in xmods:
if mod not in modlist:
tmodlist.append(mod)
# Remove any modules not producing the type requested
rtypes = args.t.split(",")
for mod in tmodlist:
for r in rtypes:
if not sfModules[mod]['provides']:
continue
if r in sfModules[mod].get('provides',
[]) and mod not in modlist:
modlist.append(mod)
if len(modlist) == 0:
log.error("Based on your criteria, no modules were enabled.")
sys.exit(-1)
modlist += ["sfp__stor_db", "sfp__stor_stdout"]
# Run the scan
if sfConfig['__logging']:
log.info(f"Modules enabled ({len(modlist)}): {','.join(modlist)}")
cfg = sf.configUnserialize(dbh.configGet(), sfConfig)
# Debug mode is a variable that gets stored to the DB, so re-apply it
if args.debug:
cfg['_debug'] = True
else:
cfg['_debug'] = False
# If strict mode is enabled, filter the output from modules.
if args.x and args.t:
cfg['__outputfilter'] = args.t.split(",")
# Prepare scan output headers
if args.o == "json":
print("[", end='')
elif not args.H:
delim = "\t"
if args.o == "tab":
delim = "\t"
if args.o == "csv":
if args.D:
delim = args.D
else:
delim = ","
if args.r:
if delim == "\t":
headers = '{0:30}{1}{2:45}{3}{4}{5}{6}'.format(
"Source", delim, "Type", delim, "Source Data", delim,
"Data")
else:
headers = delim.join(["Source", "Type", "Source Data", "Data"])
else:
if delim == "\t":
headers = '{0:30}{1}{2:45}{3}{4}'.format(
"Source", delim, "Type", delim, "Data")
else:
headers = delim.join(["Source", "Type", "Data"])
print(headers)
# Start running a new scan
scanName = target
scanId = sf.genScanInstanceId()
try:
p = mp.Process(target=SpiderFootScanner,
args=(scanName, scanId, target, targetType, modlist,
cfg))
p.daemon = True
p.start()
except BaseException as e:
log.error(f"Scan [{scanId}] failed: {e}")
sys.exit(-1)
# Poll for scan status until completion
while True:
time.sleep(1)
info = dbh.scanInstanceGet(scanId)
if not info:
continue
if info[5] in [
"ERROR-FAILED", "ABORT-REQUESTED", "ABORTED", "FINISHED"
]:
if sfConfig['__logging']:
log.info(f"Scan completed with status {info[5]}")
if args.o == "json":
print("]")
sys.exit(0)
return
def test_setup(self):
sf = SpiderFoot(self.default_options)
module = sfp_whoisology()
module.setup(sf, dict())
def main():
if len(sys.argv) <= 1:
print(
"SpiderFoot requires -l <ip>:<port> to start the web server. Try --help for guidance."
)
sys.exit(-1)
# web server config
sfWebUiConfig = {'host': '127.0.0.1', 'port': 5001, 'root': '/'}
# 'Global' configuration options
# These can be overriden on a per-module basis, and some will
# be overridden from saved configuration settings stored in the DB.
sfConfig = {
'_debug': False, # Debug
'__logging': True, # Logging in general
'__outputfilter': None, # Event types to filter from modules' output
'_useragent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0', # User-Agent to use for HTTP requests
'_dnsserver': '', # Override the default resolver
'_fetchtimeout': 5, # number of seconds before giving up on a fetch
'_internettlds':
'https://publicsuffix.org/list/effective_tld_names.dat',
'_internettlds_cache': 72,
'_genericusers':
"abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",
'__version__': '3.3-DEV',
'__database': 'spiderfoot.db',
'__modules__': None, # List of modules. Will be set after start-up.
'_socks1type': '',
'_socks2addr': '',
'_socks3port': '',
'_socks4user': '',
'_socks5pwd': '',
'_torctlport': 9051
}
sfOptdescs = {
'_debug': "Enable debugging?",
'_useragent':
"User-Agent string to use for HTTP requests. Prefix with an '@' to randomly select the User Agent from a file containing user agent strings for each request, e.g. @C:\\useragents.txt or @/home/bob/useragents.txt. Or supply a URL to load the list from there.",
'_dnsserver':
"Override the default resolver with another DNS server. For example, 8.8.8.8 is Google's open DNS server.",
'_fetchtimeout':
"Number of seconds before giving up on a HTTP request.",
'_internettlds': "List of Internet TLDs.",
'_internettlds_cache':
"Hours to cache the Internet TLD list. This can safely be quite a long time given that the list doesn't change too often.",
'_genericusers':
"List of usernames that if found as usernames or as part of e-mail addresses, should be treated differently to non-generics.",
'_socks1type': "SOCKS Server Type. Can be '4', '5', 'HTTP' or 'TOR'",
'_socks2addr': 'SOCKS Server IP Address.',
'_socks3port':
'SOCKS Server TCP Port. Usually 1080 for 4/5, 8080 for HTTP and 9050 for TOR.',
'_socks4user':
'******',
'_socks5pwd': "SOCKS Password. Valid only for SOCKS5 servers.",
'_torctlport':
"The port TOR is taking control commands on. This is necessary for SpiderFoot to tell TOR to re-circuit when it suspects anonymity is compromised.",
'_modulesenabled':
"Modules enabled for the scan." # This is a hack to get a description for an option not actually available.
}
# Legacy way to run the server
args = None
p = argparse.ArgumentParser(
description='SpiderFoot 3.3-DEV: Open Source Intelligence Automation.')
p.add_argument("-d",
"--debug",
action='store_true',
help="Enable debug output.")
p.add_argument("-l", metavar="IP:port", help="IP and port to listen on.")
p.add_argument("-m",
metavar="mod1,mod2,...",
type=str,
help="Modules to enable.")
p.add_argument("-M",
"--modules",
action='store_true',
help="List available modules.")
p.add_argument("-s", metavar="TARGET", help="Target for the scan.")
p.add_argument(
"-t",
metavar="type1,type2,...",
type=str,
help="Event types to collect (modules selected automatically).")
p.add_argument("-T",
"--types",
action='store_true',
help="List available event types.")
p.add_argument(
"-o",
metavar="tab|csv|json",
type=str,
help="Output format. Tab is default. If using json, -q is enforced.")
p.add_argument("-H",
action='store_true',
help="Don't print field headers, just data.")
p.add_argument("-n", action='store_true', help="Strip newlines from data.")
p.add_argument("-r",
action='store_true',
help="Include the source data field in tab/csv output.")
p.add_argument(
"-S",
metavar="LENGTH",
type=int,
help="Maximum data length to display. By default, all data is shown.")
p.add_argument("-D",
metavar='DELIMITER',
type=str,
help="Delimiter to use for CSV output. Default is ,.")
p.add_argument(
"-f",
action='store_true',
help="Filter out other event types that weren't requested with -t.")
p.add_argument("-F",
metavar="type1,type2,...",
type=str,
help="Show only a set of event types, comma-separated.")
p.add_argument(
"-x",
action='store_true',
help=
"STRICT MODE. Will only enable modules that can directly consume your target, and if -t was specified only those events will be consumed by modules. This overrides -t and -m options."
)
p.add_argument("-q",
action='store_true',
help="Disable logging. This will also hide errors!")
args = p.parse_args()
if args.debug:
sfConfig['_debug'] = True
log.setLevel(logging.DEBUG)
else:
log.setLevel(logging.INFO)
sfConfig['_debug'] = False
if args.q or args.o == "json":
log.setLevel(logging.NOTSET)
sfConfig['__logging'] = False
sfModules = dict()
sft = SpiderFoot(sfConfig)
# Load each module in the modules directory with a .py extension
mod_dir = sft.myPath() + '/modules/'
if not os.path.isdir(mod_dir):
log.critical(f"Modules directory does not exist: {mod_dir}")
sys.exit(-1)
for filename in os.listdir(mod_dir):
if filename.startswith("sfp_") and filename.endswith(".py"):
# Skip the module template and debugging modules
if filename in ('sfp_template.py', 'sfp_stor_print.py'):
continue
modName = filename.split('.')[0]
# Load and instantiate the module
sfModules[modName] = dict()
mod = __import__('modules.' + modName, globals(), locals(),
[modName])
sfModules[modName]['object'] = getattr(mod, modName)()
try:
sfModules[modName]['name'] = sfModules[modName]['object'].meta[
'name']
sfModules[modName]['cats'] = sfModules[modName][
'object'].meta.get('categories', list())
sfModules[modName]['group'] = sfModules[modName][
'object'].meta.get('useCases', list())
if len(sfModules[modName]['cats']) > 1:
raise ValueError(
f"Module {modName} has multiple categories defined but only one is supported."
)
sfModules[modName]['labels'] = sfModules[modName][
'object'].meta.get('flags', list())
sfModules[modName]['descr'] = sfModules[modName][
'object'].meta['summary']
sfModules[modName]['provides'] = sfModules[modName][
'object'].producedEvents()
sfModules[modName]['consumes'] = sfModules[modName][
'object'].watchedEvents()
sfModules[modName]['meta'] = sfModules[modName]['object'].meta
if hasattr(sfModules[modName]['object'], 'opts'):
sfModules[modName]['opts'] = sfModules[modName][
'object'].opts
if hasattr(sfModules[modName]['object'], 'optdescs'):
sfModules[modName]['optdescs'] = sfModules[modName][
'object'].optdescs
except BaseException as e:
log.critical(f"Failed to load {modName}: {e}")
sys.exit(-1)
if not sfModules:
log.critical(f"No modules found in modules directory: {mod_dir}")
sys.exit(-1)
# Add module info to sfConfig so it can be used by the UI
sfConfig['__modules__'] = sfModules
# Add descriptions of the global config options
sfConfig['__globaloptdescs__'] = sfOptdescs
if args.l:
try:
(host, port) = args.l.split(":")
except BaseException:
log.critical("Invalid ip:port format.")
sys.exit(-1)
sfWebUiConfig['host'] = host
sfWebUiConfig['port'] = port
start_web_server(sfWebUiConfig, sfConfig)
else:
start_scan(sfConfig, sfModules, args)
def test_setup(self):
sf = SpiderFoot(self.default_options)
module = sfp_socialprofiles()
module.setup(sf, dict())
def test_init_no_options(self):
"""
Test __init__(self, options, handle=None):
"""
sf = SpiderFoot(dict())
self.assertEqual('TBD', 'TBD')
def test_setup(self):
sf = SpiderFoot(self.default_options)
module = sfp_etherscan()
module.setup(sf, dict())
def test_setup(self):
sf = SpiderFoot(self.default_options)
module = sfp_neutrinoapi()
module.setup(sf, dict())
def test_setup(self):
sf = SpiderFoot(self.default_options)
module = sfp_dnsresolve()
module.setup(sf, dict())
def setup_server():
default_config = {
'_debug': False, # Debug
'__logging': True, # Logging in general
'__outputfilter':
None, # Event types to filter from modules' output
'_useragent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:62.0) Gecko/20100101 Firefox/62.0', # User-Agent to use for HTTP requests
'_dnsserver': '', # Override the default resolver
'_fetchtimeout':
5, # number of seconds before giving up on a fetch
'_internettlds':
'https://publicsuffix.org/list/effective_tld_names.dat',
'_internettlds_cache': 72,
'_genericusers':
"abuse,admin,billing,compliance,devnull,dns,ftp,hostmaster,inoc,ispfeedback,ispsupport,list-request,list,maildaemon,marketing,noc,no-reply,noreply,null,peering,peering-notify,peering-request,phish,phishing,postmaster,privacy,registrar,registry,root,routing-registry,rr,sales,security,spam,support,sysadmin,tech,undisclosed-recipients,unsubscribe,usenet,uucp,webmaster,www",
'__database': 'spiderfoot.test.db', # note: test database file
'__modules__':
None, # List of modules. Will be set after start-up.
'_socks1type': '',
'_socks2addr': '',
'_socks3port': '',
'_socks4user': '',
'_socks5pwd': '',
'_torctlport': 9051,
'__logstdout': False
}
default_web_config = {'root': '/'}
sfModules = dict()
sf = SpiderFoot(default_config)
mod_dir = sf.myPath() + '/modules/'
for filename in os.listdir(mod_dir):
if not filename.startswith("sfp_"):
continue
if not filename.endswith(".py"):
continue
# Skip the module template and debugging modules
if filename in ('sfp_template.py', 'sfp_stor_print.py'):
continue
modName = filename.split('.')[0]
# Load and instantiate the module
sfModules[modName] = dict()
mod = __import__('modules.' + modName, globals(), locals(),
[modName])
sfModules[modName]['object'] = getattr(mod, modName)()
sfModules[modName]['name'] = sfModules[modName]['object'].meta[
'name']
sfModules[modName]['cats'] = sfModules[modName]['object'].meta.get(
'categories', list())
sfModules[modName]['group'] = sfModules[modName][
'object'].meta.get('useCases', list())
if len(sfModules[modName]['cats']) > 1:
raise ValueError(
f"Module {modName} has multiple categories defined but only one is supported."
)
sfModules[modName]['labels'] = sfModules[modName][
'object'].meta.get('flags', list())
sfModules[modName]['descr'] = sfModules[modName]['object'].meta[
'summary']
sfModules[modName]['provides'] = sfModules[modName][
'object'].producedEvents()
sfModules[modName]['consumes'] = sfModules[modName][
'object'].watchedEvents()
sfModules[modName]['meta'] = sfModules[modName]['object'].meta
if hasattr(sfModules[modName]['object'], 'opts'):
sfModules[modName]['opts'] = sfModules[modName]['object'].opts
if hasattr(sfModules[modName]['object'], 'optdescs'):
sfModules[modName]['optdescs'] = sfModules[modName][
'object'].optdescs
default_config['__modules__'] = sfModules
conf = {
'/query': {
'tools.encode.text_only': False,
'tools.encode.add_charset': True,
},
'/static': {
'tools.staticdir.on': True,
'tools.staticdir.dir': 'static',
'tools.staticdir.root': sf.myPath()
}
}
cherrypy.tree.mount(SpiderFootWebUi(default_web_config,
default_config),
script_name=default_web_config.get('root'),
config=conf)
def test_setup(self):
sf = SpiderFoot(self.default_options)
module = sfp_openphish()
module.setup(sf, dict())
