def test_configGet_should_return_a_dict(self):
"""
Test configGet(self)
"""
sfdb = SpiderFootDb(self.default_options, False)
config = sfdb.configGet()
self.assertIsInstance(config, dict)
def __init__(self, web_config, config):
"""Initialize web server
Args:
web_config: config settings for web interface (interface, port, root path)
config: SpiderFoot config
Raises:
TypeError: arg type is invalid
ValueError: arg value is invalid
"""
if not isinstance(config, dict):
raise TypeError(f"config is {type(config)}; expected dict()")
if not config:
raise ValueError("config is empty")
if not isinstance(web_config, dict):
raise TypeError(f"web_config is {type(web_config)}; expected dict()")
if not config:
raise ValueError("web_config is empty")
self.docroot = web_config.get('root', '/').rstrip('/')
# 'config' supplied will be the defaults, let's supplement them
# now with any configuration which may have previously been saved.
self.defaultConfig = deepcopy(config)
dbh = SpiderFootDb(self.defaultConfig)
sf = SpiderFoot(self.defaultConfig)
self.config = sf.configUnserialize(dbh.configGet(), self.defaultConfig)
cherrypy.config.update({
'error_page.401': self.error_page_401,
'error_page.404': self.error_page_404,
'request.error_response': self.error_page
})
csp = (
secure.ContentSecurityPolicy()
.default_src("'self'")
.script_src("'self'", "'unsafe-inline'", "blob:")
.style_src("'self'", "'unsafe-inline'")
.base_uri("'self'")
.connect_src("'self'", "data:")
.frame_src("'self'", 'data:')
.img_src("'self'", "data:")
)
secure_headers = secure.Secure(
server=secure.Server().set("server"),
cache=secure.CacheControl().must_revalidate(),
csp=csp,
referrer=secure.ReferrerPolicy().no_referrer(),
)
cherrypy.config.update({
"tools.response_headers.on": True,
"tools.response_headers.headers": secure_headers.framework.cherrypy()
})
def __init__(self, config):
"""Initialize web server
Args:
config: TBD
"""
if not isinstance(config, dict):
raise TypeError("config is %s; expected dict()" % type(config))
if not config:
raise ValueError("config is empty")
self.defaultConfig = deepcopy(config)
dbh = SpiderFootDb(self.defaultConfig)
# 'config' supplied will be the defaults, let's supplement them
# now with any configuration which may have previously been
# saved.
sf = SpiderFoot(self.defaultConfig)
self.config = sf.configUnserialize(dbh.configGet(), self.defaultConfig)
if self.config['__webaddr'] == "0.0.0.0": # nosec
addr = "<IP of this host>"
else:
addr = self.config['__webaddr']
self.docroot = self.config['__docroot'].rstrip('/')
cherrypy.config.update({
'error_page.404': self.error_page_404,
'request.error_response': self.error_page
})
secure_headers = SecureHeaders(
server="server",
cache=False,
csp="default-src 'self' ; script-src 'self' 'unsafe-inline' blob: ; style-src 'self' 'unsafe-inline' ; img-src 'self' data:"
)
cherrypy.config.update({
"tools.response_headers.on": True,
"tools.response_headers.headers": secure_headers.cherrypy()
})
if (cherrypy.server.ssl_certificate is None or cherrypy.server.ssl_private_key is None):
url = "http://%s:%s%s" % (addr, self.config['__webport'], self.docroot)
else:
url = "https://%s:%s%s" % (addr, self.config['__webport'], self.docroot)
print("")
print("")
print("*************************************************************")
print(" Use SpiderFoot by starting your web browser of choice and ")
print(" browse to %s" % url)
print("*************************************************************")
print("")
print("")
def test_configClear_should_clear_config(self):
"""
Test configClear(self)
"""
sfdb = SpiderFootDb(self.default_options, False)
opts = dict()
opts['example'] = 'example non-default config opt'
sfdb.configSet(opts)
config = sfdb.configGet()
self.assertIsInstance(config, dict)
self.assertIn('example', config)
sfdb.configClear()
config = sfdb.configGet()
self.assertIsInstance(config, dict)
self.assertNotIn('example', config)
def test_configSet_should_set_config_opts(self):
"""
Test configSet(self, optMap=dict())
"""
sfdb = SpiderFootDb(self.default_options, False)
opts = dict()
opts['example'] = 'example non-default config opt'
sfdb.configSet(opts)
config = sfdb.configGet()
self.assertIsInstance(config, dict)
self.assertIn('example', config)
self.assertEqual('TBD', 'TBD')
def __init__(self, web_config, config):
"""Initialize web server
Args:
web_config: config settings for web interface (interface, port, root path)
config: SpiderFoot config
"""
if not isinstance(config, dict):
raise TypeError(f"config is {type(config)}; expected dict()")
if not config:
raise ValueError("config is empty")
if not isinstance(web_config, dict):
raise TypeError(f"web_config is {type(web_config)}; expected dict()")
if not config:
raise ValueError("web_config is empty")
self.docroot = web_config.get('root', '/').rstrip('/')
# 'config' supplied will be the defaults, let's supplement them
# now with any configuration which may have previously been saved.
self.defaultConfig = deepcopy(config)
dbh = SpiderFootDb(self.defaultConfig)
sf = SpiderFoot(self.defaultConfig)
self.config = sf.configUnserialize(dbh.configGet(), self.defaultConfig)
cherrypy.config.update({
'error_page.404': self.error_page_404,
'request.error_response': self.error_page
})
secure_headers = SecureHeaders(
server="server",
cache=False,
csp="default-src 'self' ; script-src 'self' 'unsafe-inline' blob: ; style-src 'self' 'unsafe-inline' ; img-src 'self' data:"
)
cherrypy.config.update({
"tools.response_headers.on": True,
"tools.response_headers.headers": secure_headers.cherrypy()
})
def start_scan(sfConfig, sfModules, args):
global dbh
global scanId
dbh = SpiderFootDb(sfConfig, init=True)
sf = SpiderFoot(sfConfig)
if args.modules:
log.info("Modules available:")
for m in sorted(sfModules.keys()):
if "__" in m:
continue
print(('{0:25} {1}'.format(m, sfModules[m]['descr'])))
sys.exit(0)
if args.types:
log.info("Types available:")
typedata = dbh.eventTypes()
types = dict()
for r in typedata:
types[r[1]] = r[0]
for t in sorted(types.keys()):
print(('{0:45} {1}'.format(t, types[t])))
sys.exit(0)
if not args.s:
log.error(
"You must specify a target when running in scan mode. Try --help for guidance."
)
sys.exit(-1)
if args.x and not args.t:
log.error("-x can only be used with -t. Use --help for guidance.")
sys.exit(-1)
if args.x and args.m:
log.error(
"-x can only be used with -t and not with -m. Use --help for guidance."
)
sys.exit(-1)
if args.r and (args.o and args.o not in ["tab", "csv"]):
log.error("-r can only be used when your output format is tab or csv.")
sys.exit(-1)
if args.H and (args.o and args.o not in ["tab", "csv"]):
log.error("-H can only be used when your output format is tab or csv.")
sys.exit(-1)
if args.D and args.o != "csv":
log.error("-D can only be used when using the csv output format.")
sys.exit(-1)
target = args.s
# Usernames and names - quoted on the commandline - won't have quotes,
# so add them.
if " " in target:
target = f"\"{target}\""
if "." not in target and not target.startswith("+") and '"' not in target:
target = f"\"{target}\""
targetType = sf.targetType(target)
if not targetType:
log.error(f"Could not determine target type. Invalid target: {target}")
sys.exit(-1)
target = target.strip('"')
modlist = list()
if not args.t and not args.m:
log.warning(
"You didn't specify any modules or types, so all will be enabled.")
for m in list(sfModules.keys()):
if "__" in m:
continue
modlist.append(m)
signal.signal(signal.SIGINT, handle_abort)
# If the user is scanning by type..
# 1. Find modules producing that type
if args.t:
types = args.t
modlist = sf.modulesProducing(types)
newmods = deepcopy(modlist)
newmodcpy = deepcopy(newmods)
# 2. For each type those modules consume, get modules producing
while len(newmodcpy) > 0:
for etype in sf.eventsToModules(newmodcpy):
xmods = sf.modulesProducing([etype])
for mod in xmods:
if mod not in modlist:
modlist.append(mod)
newmods.append(mod)
newmodcpy = deepcopy(newmods)
newmods = list()
# Easier if scanning by module
if args.m:
modlist = list(filter(None, args.m.split(",")))
# Add sfp__stor_stdout to the module list
typedata = dbh.eventTypes()
types = dict()
for r in typedata:
types[r[1]] = r[0]
sfp__stor_stdout_opts = sfConfig['__modules__']['sfp__stor_stdout']['opts']
sfp__stor_stdout_opts['_eventtypes'] = types
if args.f:
if args.f and not args.t:
log.error("You can only use -f with -t. Use --help for guidance.")
sys.exit(-1)
sfp__stor_stdout_opts['_showonlyrequested'] = True
if args.F:
sfp__stor_stdout_opts['_requested'] = args.F.split(",")
sfp__stor_stdout_opts['_showonlyrequested'] = True
if args.o:
sfp__stor_stdout_opts['_format'] = args.o
if args.t:
sfp__stor_stdout_opts['_requested'] = args.t.split(",")
if args.n:
sfp__stor_stdout_opts['_stripnewline'] = True
if args.r:
sfp__stor_stdout_opts['_showsource'] = True
if args.S:
sfp__stor_stdout_opts['_maxlength'] = args.S
if args.D:
sfp__stor_stdout_opts['_csvdelim'] = args.D
if args.x:
tmodlist = list()
modlist = list()
xmods = sf.modulesConsuming([targetType])
for mod in xmods:
if mod not in modlist:
tmodlist.append(mod)
# Remove any modules not producing the type requested
rtypes = args.t.split(",")
for mod in tmodlist:
for r in rtypes:
if not sfModules[mod]['provides']:
continue
if r in sfModules[mod].get('provides',
[]) and mod not in modlist:
modlist.append(mod)
if len(modlist) == 0:
log.error("Based on your criteria, no modules were enabled.")
sys.exit(-1)
modlist += ["sfp__stor_db", "sfp__stor_stdout"]
# Run the scan
if sfConfig['__logging']:
log.info(f"Modules enabled ({len(modlist)}): {','.join(modlist)}")
cfg = sf.configUnserialize(dbh.configGet(), sfConfig)
# Debug mode is a variable that gets stored to the DB, so re-apply it
if args.debug:
cfg['_debug'] = True
else:
cfg['_debug'] = False
# If strict mode is enabled, filter the output from modules.
if args.x and args.t:
cfg['__outputfilter'] = args.t.split(",")
if args.o == "json":
print("[", end='')
# Start running a new scan
scanName = target
scanId = sf.genScanInstanceId()
try:
p = mp.Process(target=SpiderFootScanner,
args=(scanName, scanId, target, targetType, modlist,
cfg))
p.daemon = True
p.start()
except BaseException as e:
log.error(f"Scan [{scanId}] failed: {e}")
sys.exit(-1)
# If field headers weren't disabled, print them
if not args.H and args.o != "json":
if args.D:
delim = args.D
else:
if args.o in ["tab", None]:
delim = "\t"
if args.o == "csv":
delim = ","
if not args.r:
if delim != "\t":
print(delim.join(["Source", "Type", "Data"]))
else:
print('{0:30}{1}{2:45}{3}{4}'.format("Source", delim, "Type",
delim, "Data"))
else:
if delim != "\t":
print(delim.join(["Source", "Type", "Source Data", "Data"]))
else:
print('{0:30}{1}{2:45}{3}{4}{5}{6}'.format(
"Source", delim, "Type", delim, "Source Data", delim,
"Data"))
while True:
time.sleep(1)
info = dbh.scanInstanceGet(scanId)
if not info:
continue
if info[5] in [
"ERROR-FAILED", "ABORT-REQUESTED", "ABORTED", "FINISHED"
]:
if sfConfig['__logging']:
log.info(f"Scan completed with status {info[5]}")
if args.o == "json":
print("]")
sys.exit(0)
return None
def start_scan(sfConfig: dict, sfModules: dict, args, loggingQueue) -> None:
"""Start scan
Args:
sfConfig (dict): SpiderFoot config options
sfModules (dict): modules
args (argparse.Namespace): command line args
loggingQueue (Queue): main SpiderFoot logging queue
"""
log = logging.getLogger(f"spiderfoot.{__name__}")
global dbh
global scanId
dbh = SpiderFootDb(sfConfig, init=True)
sf = SpiderFoot(sfConfig)
if not args.s:
log.error(
"You must specify a target when running in scan mode. Try --help for guidance."
)
sys.exit(-1)
if args.x and not args.t:
log.error("-x can only be used with -t. Use --help for guidance.")
sys.exit(-1)
if args.x and args.m:
log.error(
"-x can only be used with -t and not with -m. Use --help for guidance."
)
sys.exit(-1)
if args.r and (args.o and args.o not in ["tab", "csv"]):
log.error("-r can only be used when your output format is tab or csv.")
sys.exit(-1)
if args.H and (args.o and args.o not in ["tab", "csv"]):
log.error("-H can only be used when your output format is tab or csv.")
sys.exit(-1)
if args.D and args.o != "csv":
log.error("-D can only be used when using the csv output format.")
sys.exit(-1)
target = args.s
# Usernames and names - quoted on the commandline - won't have quotes,
# so add them.
if " " in target:
target = f"\"{target}\""
if "." not in target and not target.startswith("+") and '"' not in target:
target = f"\"{target}\""
targetType = SpiderFootHelpers.targetTypeFromString(target)
if not targetType:
log.error(f"Could not determine target type. Invalid target: {target}")
sys.exit(-1)
target = target.strip('"')
modlist = list()
if not args.t and not args.m and not args.u:
log.warning(
"You didn't specify any modules, types or use case, so all modules will be enabled."
)
for m in list(sfModules.keys()):
if "__" in m:
continue
modlist.append(m)
signal.signal(signal.SIGINT, handle_abort)
# If the user is scanning by type..
# 1. Find modules producing that type
if args.t:
types = args.t
modlist = sf.modulesProducing(types)
newmods = deepcopy(modlist)
newmodcpy = deepcopy(newmods)
# 2. For each type those modules consume, get modules producing
while len(newmodcpy) > 0:
for etype in sf.eventsToModules(newmodcpy):
xmods = sf.modulesProducing([etype])
for mod in xmods:
if mod not in modlist:
modlist.append(mod)
newmods.append(mod)
newmodcpy = deepcopy(newmods)
newmods = list()
# Easier if scanning by module
if args.m:
modlist = list(filter(None, args.m.split(",")))
# Select modules if the user selected usercase
if args.u:
usecase = args.u[0].upper() + args.u[
1:] # Make the first Letter Uppercase
for mod in sfConfig['__modules__']:
if usecase == 'All' or usecase in sfConfig['__modules__'][mod][
'group']:
modlist.append(mod)
# Add sfp__stor_stdout to the module list
typedata = dbh.eventTypes()
types = dict()
for r in typedata:
types[r[1]] = r[0]
sfp__stor_stdout_opts = sfConfig['__modules__']['sfp__stor_stdout']['opts']
sfp__stor_stdout_opts['_eventtypes'] = types
if args.f:
if args.f and not args.t:
log.error("You can only use -f with -t. Use --help for guidance.")
sys.exit(-1)
sfp__stor_stdout_opts['_showonlyrequested'] = True
if args.F:
sfp__stor_stdout_opts['_requested'] = args.F.split(",")
sfp__stor_stdout_opts['_showonlyrequested'] = True
if args.o:
if args.o not in ["tab", "csv", "json"]:
log.error(
"Invalid output format selected. Must be 'tab', 'csv' or 'json'."
)
sys.exit(-1)
sfp__stor_stdout_opts['_format'] = args.o
if args.t:
sfp__stor_stdout_opts['_requested'] = args.t.split(",")
if args.n:
sfp__stor_stdout_opts['_stripnewline'] = True
if args.r:
sfp__stor_stdout_opts['_showsource'] = True
if args.S:
sfp__stor_stdout_opts['_maxlength'] = args.S
if args.D:
sfp__stor_stdout_opts['_csvdelim'] = args.D
if args.x:
tmodlist = list()
modlist = list()
xmods = sf.modulesConsuming([targetType])
for mod in xmods:
if mod not in modlist:
tmodlist.append(mod)
# Remove any modules not producing the type requested
rtypes = args.t.split(",")
for mod in tmodlist:
for r in rtypes:
if not sfModules[mod]['provides']:
continue
if r in sfModules[mod].get('provides',
[]) and mod not in modlist:
modlist.append(mod)
if len(modlist) == 0:
log.error("Based on your criteria, no modules were enabled.")
sys.exit(-1)
modlist += ["sfp__stor_db", "sfp__stor_stdout"]
if sfConfig['__logging']:
log.info(f"Modules enabled ({len(modlist)}): {','.join(modlist)}")
cfg = sf.configUnserialize(dbh.configGet(), sfConfig)
# Debug mode is a variable that gets stored to the DB, so re-apply it
if args.debug:
cfg['_debug'] = True
else:
cfg['_debug'] = False
# If strict mode is enabled, filter the output from modules.
if args.x and args.t:
cfg['__outputfilter'] = args.t.split(",")
# Prepare scan output headers
if args.o == "json":
print("[", end='')
elif not args.H:
delim = "\t"
if args.o == "tab":
delim = "\t"
if args.o == "csv":
if args.D:
delim = args.D
else:
delim = ","
if args.r:
if delim == "\t":
headers = '{0:30}{1}{2:45}{3}{4}{5}{6}'.format(
"Source", delim, "Type", delim, "Source Data", delim,
"Data")
else:
headers = delim.join(["Source", "Type", "Source Data", "Data"])
else:
if delim == "\t":
headers = '{0:30}{1}{2:45}{3}{4}'.format(
"Source", delim, "Type", delim, "Data")
else:
headers = delim.join(["Source", "Type", "Data"])
print(headers)
# Start running a new scan
scanName = target
scanId = SpiderFootHelpers.genScanInstanceId()
try:
p = mp.Process(target=startSpiderFootScanner,
args=(loggingQueue, scanName, scanId, target,
targetType, modlist, cfg))
p.daemon = True
p.start()
except BaseException as e:
log.error(f"Scan [{scanId}] failed: {e}")
sys.exit(-1)
# Poll for scan status until completion
while True:
time.sleep(1)
info = dbh.scanInstanceGet(scanId)
if not info:
continue
if info[5] in [
"ERROR-FAILED", "ABORT-REQUESTED", "ABORTED", "FINISHED"
]:
if sfConfig['__logging']:
log.info(f"Scan completed with status {info[5]}")
if args.o == "json":
print("]")
sys.exit(0)
return
if not sfModules[mod]['provides']:
continue
if r in sfModules[mod].get('provides',
[]) and mod not in modlist:
modlist.append(mod)
if len(modlist) == 0:
log.error("Based on your criteria, no modules were enabled.")
sys.exit(-1)
modlist += ["sfp__stor_db", "sfp__stor_stdout"]
# Run the scan
if sfConfig['__logging']:
log.info(f"Modules enabled ({len(modlist)}): {','.join(modlist)}")
cfg = sf.configUnserialize(dbh.configGet(), sfConfig)
# Debug mode is a variable that gets stored to the DB, so re-apply it
if args.debug:
cfg['_debug'] = True
else:
cfg['_debug'] = False
# If strict mode is enabled, filter the output from modules.
if args.x and args.t:
cfg['__outputfilter'] = args.t.split(",")
if args.o == "json":
print("[", end='')
# Start running a new scan
