def test_target_type(self):
"""
Test targetType(target)
"""
target_type = SpiderFootHelpers.targetTypeFromString("0.0.0.0")
self.assertEqual('IP_ADDRESS', target_type)
target_type = SpiderFootHelpers.targetTypeFromString(
"*****@*****.**")
self.assertEqual('EMAILADDR', target_type)
target_type = SpiderFootHelpers.targetTypeFromString("0.0.0.0/0")
self.assertEqual('NETBLOCK_OWNER', target_type)
target_type = SpiderFootHelpers.targetTypeFromString("+1234567890")
self.assertEqual('PHONE_NUMBER', target_type)
target_type = SpiderFootHelpers.targetTypeFromString('"Human Name"')
self.assertEqual('HUMAN_NAME', target_type)
target_type = SpiderFootHelpers.targetTypeFromString('"abc123"')
self.assertEqual('USERNAME', target_type)
target_type = SpiderFootHelpers.targetTypeFromString("1234567890")
self.assertEqual('BGP_AS_OWNER', target_type)
target_type = SpiderFootHelpers.targetTypeFromString("::1")
self.assertEqual('IPV6_ADDRESS', target_type)
target_type = SpiderFootHelpers.targetTypeFromString("spiderfoot.net")
self.assertEqual('INTERNET_NAME', target_type)
target_type = SpiderFootHelpers.targetTypeFromString(
"1HesYJSP1QqcyPEjnQ9vzBL1wujruNGe7R")
self.assertEqual('BITCOIN_ADDRESS', target_type)
def rerunscan(self, id):
"""Rerun a scan
Args:
id (str): scan ID
Returns:
None
Raises:
HTTPRedirect: redirect to info page for new scan
"""
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modlist = list()
dbh = SpiderFootDb(cfg)
info = dbh.scanInstanceGet(id)
if not info:
return self.error("Invalid scan ID.")
scanname = info[0]
scantarget = info[1]
scanconfig = dbh.scanConfigGet(id)
if not scanconfig:
return self.error(f"Error loading config from scan: {id}")
modlist = scanconfig['_modulesenabled'].split(',')
if "sfp__stor_stdout" in modlist:
modlist.remove("sfp__stor_stdout")
targetType = SpiderFootHelpers.targetTypeFromString(scantarget)
if not targetType:
# It must then be a name, as a re-run scan should always have a clean
# target. Put quotes around the target value and try to determine the
# target type again.
targetType = SpiderFootHelpers.targetTypeFromString(f'"{scantarget}"')
if targetType not in ["HUMAN_NAME", "BITCOIN_ADDRESS"]:
scantarget = scantarget.lower()
# Start running a new scan
scanId = SpiderFootHelpers.genScanInstanceId()
try:
p = mp.Process(target=SpiderFootScanner, args=(scanname, scanId, scantarget, targetType, modlist, cfg))
p.daemon = True
p.start()
except Exception as e:
self.log.error(f"[-] Scan [{scanId}] failed: {e}")
return self.error(f"[-] Scan [{scanId}] failed: {e}")
# Wait until the scan has initialized
while dbh.scanInstanceGet(scanId) is None:
self.log.info("Waiting for the scan to initialize...")
time.sleep(1)
raise cherrypy.HTTPRedirect(f"{self.docroot}/scaninfo?id={scanId}", status=302)
def test_target_type_invalid_seed_should_return_none(self):
"""
Test targetType(target)
"""
target_type = SpiderFootHelpers.targetTypeFromString(None)
self.assertEqual(None, target_type)
target_type = SpiderFootHelpers.targetTypeFromString("")
self.assertEqual(None, target_type)
target_type = SpiderFootHelpers.targetTypeFromString('""')
self.assertEqual(None, target_type)
def rerunscanmulti(self, ids):
"""Rerun scans
Args:
ids (str): comma separated list of scan IDs
Returns:
None
"""
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
modlist = list()
dbh = SpiderFootDb(cfg)
for id in ids.split(","):
info = dbh.scanInstanceGet(id)
if not info:
return self.error("Invalid scan ID.")
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if len(scanconfig) == 0:
return self.error("Something went wrong internally.")
modlist = scanconfig['_modulesenabled'].split(',')
if "sfp__stor_stdout" in modlist:
modlist.remove("sfp__stor_stdout")
targetType = SpiderFootHelpers.targetTypeFromString(scantarget)
if targetType is None:
# Should never be triggered for a re-run scan..
return self.error("Invalid target type. Could not recognize it as a target SpiderFoot supports.")
# Start running a new scan
scanId = SpiderFootHelpers.genScanInstanceId()
try:
p = mp.Process(target=SpiderFootScanner, args=(scanname, scanId, scantarget, targetType, modlist, cfg))
p.daemon = True
p.start()
except Exception as e:
self.log.error(f"[-] Scan [{scanId}] failed: {e}")
return self.error(f"[-] Scan [{scanId}] failed: {e}")
# Wait until the scan has initialized
while dbh.scanInstanceGet(scanId) is None:
self.log.info("Waiting for the scan to initialize...")
time.sleep(1)
templ = Template(filename='spiderfoot/templates/scanlist.tmpl', lookup=self.lookup)
return templ.render(rerunscans=True, docroot=self.docroot, pageid="SCANLIST", version=__version__)
def clonescan(self, id):
"""
Clone an existing scan (pre-selected options in the newscan page)
Args:
id (str): scan ID to clone
Returns:
None
"""
dbh = SpiderFootDb(self.config)
types = dbh.eventTypes()
info = dbh.scanInstanceGet(id)
if not info:
return self.error("Invalid scan ID.")
scanconfig = dbh.scanConfigGet(id)
scanname = info[0]
scantarget = info[1]
targetType = None
if scanname == "" or scantarget == "" or len(scanconfig) == 0:
return self.error("Something went wrong internally.")
targetType = SpiderFootHelpers.targetTypeFromString(scantarget)
if targetType is None:
# It must be a name, so wrap quotes around it
scantarget = """ + scantarget + """
modlist = scanconfig['_modulesenabled'].split(',')
templ = Template(filename='spiderfoot/templates/newscan.tmpl', lookup=self.lookup)
return templ.render(pageid='NEWSCAN', types=types, docroot=self.docroot,
modules=self.config['__modules__'], selectedmods=modlist,
scanname=str(scanname),
scantarget=str(scantarget), version=__version__)
def start_scan(sfConfig: dict, sfModules: dict, args, loggingQueue) -> None:
"""Start scan
Args:
sfConfig (dict): SpiderFoot config options
sfModules (dict): modules
args (argparse.Namespace): command line args
loggingQueue (Queue): main SpiderFoot logging queue
"""
log = logging.getLogger(f"spiderfoot.{__name__}")
global dbh
global scanId
dbh = SpiderFootDb(sfConfig, init=True)
sf = SpiderFoot(sfConfig)
if not args.s:
log.error(
"You must specify a target when running in scan mode. Try --help for guidance."
)
sys.exit(-1)
if args.x and not args.t:
log.error("-x can only be used with -t. Use --help for guidance.")
sys.exit(-1)
if args.x and args.m:
log.error(
"-x can only be used with -t and not with -m. Use --help for guidance."
)
sys.exit(-1)
if args.r and (args.o and args.o not in ["tab", "csv"]):
log.error("-r can only be used when your output format is tab or csv.")
sys.exit(-1)
if args.H and (args.o and args.o not in ["tab", "csv"]):
log.error("-H can only be used when your output format is tab or csv.")
sys.exit(-1)
if args.D and args.o != "csv":
log.error("-D can only be used when using the csv output format.")
sys.exit(-1)
target = args.s
# Usernames and names - quoted on the commandline - won't have quotes,
# so add them.
if " " in target:
target = f"\"{target}\""
if "." not in target and not target.startswith("+") and '"' not in target:
target = f"\"{target}\""
targetType = SpiderFootHelpers.targetTypeFromString(target)
if not targetType:
log.error(f"Could not determine target type. Invalid target: {target}")
sys.exit(-1)
target = target.strip('"')
modlist = list()
if not args.t and not args.m and not args.u:
log.warning(
"You didn't specify any modules, types or use case, so all modules will be enabled."
)
for m in list(sfModules.keys()):
if "__" in m:
continue
modlist.append(m)
signal.signal(signal.SIGINT, handle_abort)
# If the user is scanning by type..
# 1. Find modules producing that type
if args.t:
types = args.t
modlist = sf.modulesProducing(types)
newmods = deepcopy(modlist)
newmodcpy = deepcopy(newmods)
# 2. For each type those modules consume, get modules producing
while len(newmodcpy) > 0:
for etype in sf.eventsToModules(newmodcpy):
xmods = sf.modulesProducing([etype])
for mod in xmods:
if mod not in modlist:
modlist.append(mod)
newmods.append(mod)
newmodcpy = deepcopy(newmods)
newmods = list()
# Easier if scanning by module
if args.m:
modlist = list(filter(None, args.m.split(",")))
# Select modules if the user selected usercase
if args.u:
usecase = args.u[0].upper() + args.u[
1:] # Make the first Letter Uppercase
for mod in sfConfig['__modules__']:
if usecase == 'All' or usecase in sfConfig['__modules__'][mod][
'group']:
modlist.append(mod)
# Add sfp__stor_stdout to the module list
typedata = dbh.eventTypes()
types = dict()
for r in typedata:
types[r[1]] = r[0]
sfp__stor_stdout_opts = sfConfig['__modules__']['sfp__stor_stdout']['opts']
sfp__stor_stdout_opts['_eventtypes'] = types
if args.f:
if args.f and not args.t:
log.error("You can only use -f with -t. Use --help for guidance.")
sys.exit(-1)
sfp__stor_stdout_opts['_showonlyrequested'] = True
if args.F:
sfp__stor_stdout_opts['_requested'] = args.F.split(",")
sfp__stor_stdout_opts['_showonlyrequested'] = True
if args.o:
if args.o not in ["tab", "csv", "json"]:
log.error(
"Invalid output format selected. Must be 'tab', 'csv' or 'json'."
)
sys.exit(-1)
sfp__stor_stdout_opts['_format'] = args.o
if args.t:
sfp__stor_stdout_opts['_requested'] = args.t.split(",")
if args.n:
sfp__stor_stdout_opts['_stripnewline'] = True
if args.r:
sfp__stor_stdout_opts['_showsource'] = True
if args.S:
sfp__stor_stdout_opts['_maxlength'] = args.S
if args.D:
sfp__stor_stdout_opts['_csvdelim'] = args.D
if args.x:
tmodlist = list()
modlist = list()
xmods = sf.modulesConsuming([targetType])
for mod in xmods:
if mod not in modlist:
tmodlist.append(mod)
# Remove any modules not producing the type requested
rtypes = args.t.split(",")
for mod in tmodlist:
for r in rtypes:
if not sfModules[mod]['provides']:
continue
if r in sfModules[mod].get('provides',
[]) and mod not in modlist:
modlist.append(mod)
if len(modlist) == 0:
log.error("Based on your criteria, no modules were enabled.")
sys.exit(-1)
modlist += ["sfp__stor_db", "sfp__stor_stdout"]
if sfConfig['__logging']:
log.info(f"Modules enabled ({len(modlist)}): {','.join(modlist)}")
cfg = sf.configUnserialize(dbh.configGet(), sfConfig)
# Debug mode is a variable that gets stored to the DB, so re-apply it
if args.debug:
cfg['_debug'] = True
else:
cfg['_debug'] = False
# If strict mode is enabled, filter the output from modules.
if args.x and args.t:
cfg['__outputfilter'] = args.t.split(",")
# Prepare scan output headers
if args.o == "json":
print("[", end='')
elif not args.H:
delim = "\t"
if args.o == "tab":
delim = "\t"
if args.o == "csv":
if args.D:
delim = args.D
else:
delim = ","
if args.r:
if delim == "\t":
headers = '{0:30}{1}{2:45}{3}{4}{5}{6}'.format(
"Source", delim, "Type", delim, "Source Data", delim,
"Data")
else:
headers = delim.join(["Source", "Type", "Source Data", "Data"])
else:
if delim == "\t":
headers = '{0:30}{1}{2:45}{3}{4}'.format(
"Source", delim, "Type", delim, "Data")
else:
headers = delim.join(["Source", "Type", "Data"])
print(headers)
# Start running a new scan
scanName = target
scanId = SpiderFootHelpers.genScanInstanceId()
try:
p = mp.Process(target=startSpiderFootScanner,
args=(loggingQueue, scanName, scanId, target,
targetType, modlist, cfg))
p.daemon = True
p.start()
except BaseException as e:
log.error(f"Scan [{scanId}] failed: {e}")
sys.exit(-1)
# Poll for scan status until completion
while True:
time.sleep(1)
info = dbh.scanInstanceGet(scanId)
if not info:
continue
if info[5] in [
"ERROR-FAILED", "ABORT-REQUESTED", "ABORTED", "FINISHED"
]:
if sfConfig['__logging']:
log.info(f"Scan completed with status {info[5]}")
if args.o == "json":
print("]")
sys.exit(0)
return
def startscan(self, scanname, scantarget, modulelist, typelist, usecase):
"""Initiate a scan
Args:
scanname (str): scan name
scantarget (str): scan target
modulelist (str): comma separated list of modules to use
typelist (str): selected modules based on produced event data types
usecase (str): selected module group (passive, investigate, footprint, all)
Returns:
str: start scan status as JSON
Raises:
HTTPRedirect: redirect to new scan info page
"""
[scanname, scantarget] = self.cleanUserInput([scanname, scantarget])
if not scanname:
if cherrypy.request.headers.get('Accept') and 'application/json' in cherrypy.request.headers.get('Accept'):
cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"
return json.dumps(["ERROR", "Incorrect usage: scan name was not specified."]).encode('utf-8')
return self.error("Invalid request: scan name was not specified.")
if not scantarget:
if cherrypy.request.headers.get('Accept') and 'application/json' in cherrypy.request.headers.get('Accept'):
cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"
return json.dumps(["ERROR", "Incorrect usage: scan target was not specified."]).encode('utf-8')
return self.error("Invalid request: scan target was not specified.")
if not typelist and not modulelist and not usecase:
if cherrypy.request.headers.get('Accept') and 'application/json' in cherrypy.request.headers.get('Accept'):
cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"
return json.dumps(["ERROR", "Incorrect usage: no modules specified for scan."]).encode('utf-8')
return self.error("Invalid request: no modules specified for scan.")
targetType = SpiderFootHelpers.targetTypeFromString(scantarget)
if targetType is None:
if cherrypy.request.headers.get('Accept') and 'application/json' in cherrypy.request.headers.get('Accept'):
cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"
return json.dumps(["ERROR", "Unrecognised target type."]).encode('utf-8')
return self.error("Invalid target type. Could not recognize it as a target SpiderFoot supports.")
# Swap the globalscantable for the database handler
dbh = SpiderFootDb(self.config)
# Snapshot the current configuration to be used by the scan
cfg = deepcopy(self.config)
sf = SpiderFoot(cfg)
modlist = list()
# User selected modules
if modulelist:
modlist = modulelist.replace('module_', '').split(',')
# User selected types
if len(modlist) == 0 and typelist:
typesx = typelist.replace('type_', '').split(',')
# 1. Find all modules that produce the requested types
modlist = sf.modulesProducing(typesx)
newmods = deepcopy(modlist)
newmodcpy = deepcopy(newmods)
# 2. For each type those modules consume, get modules producing
while len(newmodcpy) > 0:
for etype in sf.eventsToModules(newmodcpy):
xmods = sf.modulesProducing([etype])
for mod in xmods:
if mod not in modlist:
modlist.append(mod)
newmods.append(mod)
newmodcpy = deepcopy(newmods)
newmods = list()
# User selected a use case
if len(modlist) == 0 and usecase:
for mod in self.config['__modules__']:
if usecase == 'all' or usecase in self.config['__modules__'][mod]['group']:
modlist.append(mod)
# If we somehow got all the way through to here and still don't have any modules selected
if not modlist:
if cherrypy.request.headers.get('Accept') and 'application/json' in cherrypy.request.headers.get('Accept'):
cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"
return json.dumps(["ERROR", "Incorrect usage: no modules specified for scan."]).encode('utf-8')
return self.error("Invalid request: no modules specified for scan.")
# Add our mandatory storage module
if "sfp__stor_db" not in modlist:
modlist.append("sfp__stor_db")
modlist.sort()
# Delete the stdout module in case it crept in
if "sfp__stor_stdout" in modlist:
modlist.remove("sfp__stor_stdout")
# Start running a new scan
if targetType in ["HUMAN_NAME", "USERNAME", "BITCOIN_ADDRESS"]:
scantarget = scantarget.replace("\"", "")
else:
scantarget = scantarget.lower()
# Start running a new scan
scanId = SpiderFootHelpers.genScanInstanceId()
try:
p = mp.Process(target=SpiderFootScanner, args=(scanname, scanId, scantarget, targetType, modlist, cfg))
p.daemon = True
p.start()
except Exception as e:
self.log.error(f"[-] Scan [{scanId}] failed: {e}")
return self.error(f"[-] Scan [{scanId}] failed: {e}")
# Wait until the scan has initialized
# Check the database for the scan status results
while dbh.scanInstanceGet(scanId) is None:
self.log.info("Waiting for the scan to initialize...")
time.sleep(1)
if cherrypy.request.headers.get('Accept') and 'application/json' in cherrypy.request.headers.get('Accept'):
cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"
return json.dumps(["SUCCESS", scanId]).encode('utf-8')
raise cherrypy.HTTPRedirect(f"{self.docroot}/scaninfo?id={scanId}")
