def rerunscanmulti(self, ids):
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
for id in ids.split(","):
info = dbh.scanInstanceGet(id)
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if len(scanconfig) == 0:
return self.error("Something went wrong internally.")
modlist = scanconfig['_modulesenabled'].split(',')
targetType = sf.targetType(scantarget)
if targetType == None:
# Should never be triggered for a re-run scan..
return self.error("Invalid target type. Could not recognize it as " + \
"a human name, IP address, IP subnet, ASN, domain name or host name.")
# Start running a new scan
newId = sf.genScanInstanceGUID(scanname)
t = SpiderFootScanner(
unicode(scanname, 'utf-8', errors='replace'),
unicode(scantarget, 'utf-8', errors='replace').lower(),
targetType, newId, modlist, cfg, modopts)
t.start()
# Wait until the scan has initialized
while globalScanStatus.getStatus(newId) == None:
print("[info] Waiting for the scan to initialize...")
time.sleep(1)
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(rerunscans=True,
docroot=self.docroot,
pageid="SCANLIST")
def test_init_argument_scanName_of_invalid_type_should_raise_TypeError(
self):
"""
Test __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts, start=True)
"""
scan_id = str(uuid.uuid4())
module_list = ['sfp__stor_db']
invalid_types = [None, list(), dict(), int()]
for invalid_type in invalid_types:
with self.subTest(invalid_type=invalid_type):
with self.assertRaises(TypeError):
SpiderFootScanner(invalid_type,
scan_id,
"spiderfoot.net",
"IP_ADDRESS",
module_list,
self.default_options,
start=False)
def test_init_argument_start_false_should_create_a_scan_without_starting_the_scan(
self):
"""
Test __init__(self, scanName, scanId, scanTarget, targetType, moduleList, globalOpts, start=True)
"""
opts = self.default_options
opts['__modules__'] = dict()
scan_id = str(uuid.uuid4())
module_list = ['sfp__stor_db']
sfscan = SpiderFootScanner("example scan name",
scan_id,
"spiderfoot.net",
"IP_ADDRESS",
module_list,
opts,
start=False)
self.assertIsInstance(sfscan, SpiderFootScanner)
self.assertEqual(sfscan.status, "INITIALIZING")
def rerunscan(self, id):
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
info = dbh.scanInstanceGet(id)
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if len(scanconfig) == 0:
return self.error("Something went wrong internally.")
modlist = scanconfig['_modulesenabled'].split(',')
targetType = sf.targetType(scantarget)
if targetType == None:
# It must then be a name, as a re-run scan should always have a clean
# target.
targetType = "HUMAN_NAME"
if targetType != "HUMAN_NAME":
scantarget = scantarget.lower()
# Start running a new scan
newId = sf.genScanInstanceGUID(scanname)
t = SpiderFootScanner(scanname, scantarget, targetType, newId,
modlist, cfg, modopts)
t.start()
# Wait until the scan has initialized
while globalScanStatus.getStatus(newId) == None:
print("[info] Waiting for the scan to initialize...")
time.sleep(1)
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
return templ.render(id=newId, name=unicode(scanname, 'utf-8', errors='replace'), docroot=self.docroot,
status=globalScanStatus.getStatus(newId), pageid="SCANLIST")
def startscan(self, scanname, scantarget, modulelist):
modopts = dict() # Not used yet as module options are set globally
[scanname, scantarget] = self.cleanUserInput([scanname, scantarget])
if scanname == "" or scantarget == "" or modulelist == "":
return self.error("Form incomplete.")
modlist = modulelist.replace('module_', '').split(',')
# For now we don't permit multiple simultaneous scans
for thread in threading.enumerate():
if thread.name.startswith("SF_"):
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(modules=self.config['__modules__'], alreadyRunning=True, runningScan=thread.name[3:])
# Start running a new scan
self.scanner = SpiderFootScanner(scanname, scantarget.lower(), modlist, self.config, modopts)
t = threading.Thread(name="SF_" + scanname, target=self.scanner.startScan)
t.start()
templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup)
return templ.render(pageid='SCANLIST',newscan=scanname)
def test__setStatus_argument_status_of_invalid_type_should_raise_TypeError(
self):
"""
Test __setStatus(self, status, started=None, ended=None)
"""
opts = self.default_options
opts['__modules__'] = dict()
scan_id = str(uuid.uuid4())
module_list = ['sfp__stor_db']
sfscan = SpiderFootScanner("example scan name",
scan_id,
"spiderfoot.net",
"IP_ADDRESS",
module_list,
opts,
start=False)
invalid_types = [None, list(), dict(), int()]
for invalid_type in invalid_types:
with self.subTest(invalid_type=invalid_type):
with self.assertRaises(TypeError):
sfscan._SpiderFootScanner__setStatus(invalid_type)
def startscan(self, scanname, scantarget, modulelist, typelist, usecase):
global globalScanStatus
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
types = dbh.eventTypes()
targetType = None
[scanname, scantarget] = self.cleanUserInput([scanname, scantarget])
if scanname == "" or scantarget == "":
return self.error("Form incomplete.")
if typelist == "" and modulelist == "" and usecase == "":
return self.error("Form incomplete.")
# User selected modules
if modulelist != "":
modlist = modulelist.replace('module_', '').split(',')
# User selected types
if len(modlist) == 0 and typelist != "":
typesx = typelist.replace('type_', '').split(',')
# 1. Find all modules that produce the requested types
modlist = sf.modulesProducing(typesx)
newmods = deepcopy(modlist)
newmodcpy = deepcopy(newmods)
# 2. For each type those modules consume, get modules producing
while len(newmodcpy) > 0:
for etype in sf.eventsToModules(newmodcpy):
xmods = sf.modulesProducing([etype])
for mod in xmods:
if mod not in modlist:
modlist.append(mod)
newmods.append(mod)
newmodcpy = deepcopy(newmods)
newmods = list()
# User selected a use case
if len(modlist) == 0 and usecase != "":
for mod in self.config['__modules__']:
if usecase == 'all' or usecase in self.config['__modules__'][
mod]['cats']:
modlist.append(mod)
# Add our mandatory storage module..
if "sfp__stor_db" not in modlist:
modlist.append("sfp__stor_db")
modlist.sort()
targetType = sf.targetType(scantarget)
if targetType is None:
return self.error("Invalid target type. Could not recognize it as " + \
"an IP address, IP subnet, domain name or host name.")
# Start running a new scan
scanId = sf.genScanInstanceGUID(scanname)
t = SpiderFootScanner(scanname, scantarget.lower(), targetType, scanId,
modlist, cfg, modopts)
t.start()
# Wait until the scan has initialized
while globalScanStatus.getStatus(scanId) is None:
print "[info] Waiting for the scan to initialize..."
time.sleep(1)
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
return templ.render(id=scanId,
name=scanname,
docroot=self.docroot,
status=globalScanStatus.getStatus(scanId),
pageid="SCANLIST")
print(("[*] Modules enabled (" + str(len(modlist)) + "): " +
",".join(modlist)))
cfg = sf.configUnserialize(dbh.configGet(), sfConfig)
scanId = sf.genScanInstanceGUID(target)
# Debug mode is a variable that gets stored to the DB, so re-apply it
if args.debug:
cfg['_debug'] = True
else:
cfg['_debug'] = False
# If strict mode is enabled, filter the output from modules.
if args.x and args.t:
cfg['__outputfilter'] = args.t.split(",")
t = SpiderFootScanner(target, target, targetType, scanId, modlist, cfg,
dict())
t.daemon = True
t.start()
# If field headers weren't disabled, print them
if not args.H and args.o != "json":
if args.D:
delim = args.D
else:
if args.o in ["tab", None]:
delim = "\t"
if args.o == "csv":
delim = ","
if not args.r:
def startscan(self, scanname, scantarget, modulelist, typelist):
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(self.config)
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
[scanname, scantarget] = self.cleanUserInput([scanname, scantarget])
if scanname == "" or scantarget == "":
return self.error("Form incomplete.")
if typelist == "" and modulelist == "":
return self.error("Form incomplete.")
if modulelist != "":
modlist = modulelist.replace('module_', '').split(',')
else:
typesx = typelist.replace('type_', '').split(',')
# 1. Find all modules that produce the requested types
modlist = sf.modulesProducing(typesx)
newmods = deepcopy(modlist)
newmodcpy = deepcopy(newmods)
# 2. For each type those modules consume, get modules producing
while len(newmodcpy) > 0:
for etype in sf.eventsToModules(newmodcpy):
xmods = sf.modulesProducing([etype])
for mod in xmods:
if mod not in modlist:
modlist.append(mod)
newmods.append(mod)
newmodcpy = deepcopy(newmods)
newmods = list()
# Add our mandatory storage module..
if "sfp__stor_db" not in modlist:
modlist.append("sfp__stor_db")
modlist.sort()
# For now we don't permit multiple simultaneous scans
for thread in threading.enumerate():
if thread.name.startswith("SF_"):
templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup)
return templ.render(modules=self.config['__modules__'],
alreadyRunning=True, runningScan=thread.name[3:],
types=types, pageid="NEWSCAN")
# Start running a new scan
self.scanner = SpiderFootScanner(scanname, scantarget.lower(), modlist,
self.config, modopts)
t = threading.Thread(name="SF_" + scanname, target=self.scanner.startScan)
t.start()
# Spin cycles waiting for the scan ID to be set
while self.scanner.myId == None:
time.sleep(1)
continue
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
return templ.render(id=self.scanner.myId, name=scanname,
status=self.scanner.status, pageid="SCANLIST")
def startscan(self, scanname, scantarget, modulelist, typelist):
global globalScanStatus
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modopts = dict() # Not used yet as module options are set globally
modlist = list()
sf = SpiderFoot(cfg)
dbh = SpiderFootDb(cfg)
types = dbh.eventTypes()
targetType = None
[scanname, scantarget] = self.cleanUserInput([scanname, scantarget])
if scanname == "" or scantarget == "":
return self.error("Form incomplete.")
if typelist == "" and modulelist == "":
return self.error("Form incomplete.")
if modulelist != "":
modlist = modulelist.replace('module_', '').split(',')
else:
typesx = typelist.replace('type_', '').split(',')
# 1. Find all modules that produce the requested types
modlist = sf.modulesProducing(typesx)
newmods = deepcopy(modlist)
newmodcpy = deepcopy(newmods)
# 2. For each type those modules consume, get modules producing
while len(newmodcpy) > 0:
for etype in sf.eventsToModules(newmodcpy):
xmods = sf.modulesProducing([etype])
for mod in xmods:
if mod not in modlist:
modlist.append(mod)
newmods.append(mod)
newmodcpy = deepcopy(newmods)
newmods = list()
# Add our mandatory storage module..
if "sfp__stor_db" not in modlist:
modlist.append("sfp__stor_db")
modlist.sort()
regexToType = {
"^\d+\.\d+\.\d+\.\d+$": "IP_ADDRESS",
"^\d+\.\d+\.\d+\.\d+/\d+$": "NETBLOCK_OWNER",
"^.[a-zA-Z\-0-9\.]+$": "INTERNET_NAME"
}
# Parse the target and set the targetType
for rx in regexToType.keys():
if re.match(rx, scantarget, re.IGNORECASE):
targetType = regexToType[rx]
break
if targetType == None:
return self.error("Invalid target type. Could not recognize it as " + \
"an IP address, IP subnet, domain name or host name.")
# Start running a new scan
scanId = sf.genScanInstanceGUID(scanname)
t = SpiderFootScanner(scanname, scantarget.lower(), targetType, scanId,
modlist, cfg, modopts)
t.start()
# Wait until the scan has initialized
while globalScanStatus.getStatus(scanId) == None:
print "[info] Waiting for the scan to initialize..."
time.sleep(1)
templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup)
return templ.render(id=scanId, name=scanname,
status=globalScanStatus.getStatus(scanId), pageid="SCANLIST")
