def test_config_unserialize_invalid_reference_point_should_raise(self):
"""
Test configUnserialize(self, opts, referencePoint, filterSystem=True)
"""
sf = SpiderFoot(dict())
with self.assertRaises(TypeError):
sf.configUnserialize(dict(), None, None)
def savesettingsraw(self, allopts, token):
if str(token) != str(self.token):
return json.dumps(["ERROR", "Invalid token (" + str(self.token) + ")."])
try:
dbh = SpiderFootDb(self.config)
# Reset config to default
if allopts == "RESET":
dbh.configClear() # Clear it in the DB
self.config = deepcopy(self.defaultConfig) # Clear in memory
else:
useropts = json.loads(allopts)
cleanopts = dict()
for opt in useropts.keys():
cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0]
currentopts = deepcopy(self.config)
# Make a new config where the user options override
# the current system config.
sf = SpiderFoot(self.config)
self.config = sf.configUnserialize(cleanopts, currentopts)
dbh.configSet(sf.configSerialize(currentopts))
except Exception as e:
return json.dumps(["ERROR", "Processing one or more of your inputs failed: " + str(e)])
return json.dumps(["SUCCESS", ""])
def __init__(self, config):
self.defaultConfig = deepcopy(config)
dbh = SpiderFootDb(self.defaultConfig)
# 'config' supplied will be the defaults, let's supplement them
# now with any configuration which may have previously been
# saved.
sf = SpiderFoot(self.defaultConfig)
self.config = sf.configUnserialize(dbh.configGet(), self.defaultConfig)
if self.config['__webaddr'] == "0.0.0.0":
addr = "<IP of this host>"
else:
addr = self.config['__webaddr']
self.docroot = self.config['__docroot'].rstrip('/')
print ""
print ""
print "*************************************************************"
print " Use SpiderFoot by starting your web browser of choice and "
print " browse to http://" + addr + ":" + str(
self.config['__webport']) + self.docroot
print "*************************************************************"
print ""
print ""
def savesettings(self, allopts, token):
if str(token) != str(self.token):
return self.error("Invalid token (" + str(self.token) + ").")
try:
dbh = SpiderFootDb(self.config)
# Reset config to default
if allopts == "RESET":
dbh.configClear() # Clear it in the DB
self.config = deepcopy(self.defaultConfig) # Clear in memory
else:
useropts = json.loads(allopts)
cleanopts = dict()
for opt in useropts.keys():
cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0]
currentopts = deepcopy(self.config)
# Make a new config where the user options override
# the current system config.
sf = SpiderFoot(self.config)
self.config = sf.configUnserialize(cleanopts, currentopts)
dbh.configSet(sf.configSerialize(currentopts))
except Exception as e:
return self.error("Processing one or more of your inputs failed: " + str(e))
templ = Template(filename='dyn/opts.tmpl', lookup=self.lookup)
self.token = random.randint(0, 99999999)
return templ.render(opts=self.config, pageid='SETTINGS', updated=True,
docroot=self.docroot, token=self.token)
def savesettingsraw(self, allopts, token):
if str(token) != str(self.token):
return json.dumps(
["ERROR", "Invalid token (" + str(self.token) + ")."])
try:
dbh = SpiderFootDb(self.config)
# Reset config to default
if allopts == "RESET":
dbh.configClear() # Clear it in the DB
self.config = deepcopy(self.defaultConfig) # Clear in memory
else:
useropts = json.loads(allopts)
cleanopts = dict()
for opt in list(useropts.keys()):
cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0]
currentopts = deepcopy(self.config)
# Make a new config where the user options override
# the current system config.
sf = SpiderFoot(self.config)
self.config = sf.configUnserialize(cleanopts, currentopts)
dbh.configSet(sf.configSerialize(self.config))
except Exception as e:
return json.dumps([
"ERROR",
"Processing one or more of your inputs failed: " + str(e)
])
return json.dumps(["SUCCESS", ""])
def savesettings(self, allopts, token):
if str(token) != str(self.token):
return self.error("Invalid token (" + str(self.token) + ").")
try:
dbh = SpiderFootDb(self.config)
# Reset config to default
if allopts == "RESET":
dbh.configClear() # Clear it in the DB
self.config = deepcopy(self.defaultConfig) # Clear in memory
else:
useropts = json.loads(allopts)
cleanopts = dict()
for opt in useropts.keys():
cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0]
currentopts = deepcopy(self.config)
# Make a new config where the user options override
# the current system config.
sf = SpiderFoot(self.config)
self.config = sf.configUnserialize(cleanopts, currentopts)
dbh.configSet(sf.configSerialize(currentopts))
except Exception as e:
return self.error(
"Processing one or more of your inputs failed: " + str(e))
templ = Template(filename='dyn/opts.tmpl', lookup=self.lookup)
self.token = random.randint(0, 99999999)
return templ.render(opts=self.config,
pageid='SETTINGS',
updated=True,
docroot=self.docroot,
token=self.token)
def __init__(self, config):
self.defaultConfig = deepcopy(config)
dbh = SpiderFootDb(self.defaultConfig)
# 'config' supplied will be the defaults, let's supplement them
# now with any configuration which may have previously been
# saved.
sf = SpiderFoot(self.defaultConfig)
self.config = sf.configUnserialize(dbh.configGet(), self.defaultConfig)
if self.config['__webaddr'] == "0.0.0.0":
addr = "<IP of this host>"
else:
addr = self.config['__webaddr']
self.docroot = self.config['__docroot'].rstrip('/')
cherrypy.config.update({
'error_page.404': self.error_page_404,
'request.error_response': self.error_page
})
print("")
print("")
print("*************************************************************")
print(" Use SpiderFoot by starting your web browser of choice and ")
print(" browse to http://" + addr + ":" +
str(self.config['__webport']) + self.docroot)
print("*************************************************************")
print("")
print("")
def __init__(self, web_config, config):
"""Initialize web server
Args:
web_config: config settings for web interface (interface, port, root path)
config: SpiderFoot config
Raises:
TypeError: arg type is invalid
ValueError: arg value is invalid
"""
if not isinstance(config, dict):
raise TypeError(f"config is {type(config)}; expected dict()")
if not config:
raise ValueError("config is empty")
if not isinstance(web_config, dict):
raise TypeError(f"web_config is {type(web_config)}; expected dict()")
if not config:
raise ValueError("web_config is empty")
self.docroot = web_config.get('root', '/').rstrip('/')
# 'config' supplied will be the defaults, let's supplement them
# now with any configuration which may have previously been saved.
self.defaultConfig = deepcopy(config)
dbh = SpiderFootDb(self.defaultConfig)
sf = SpiderFoot(self.defaultConfig)
self.config = sf.configUnserialize(dbh.configGet(), self.defaultConfig)
cherrypy.config.update({
'error_page.401': self.error_page_401,
'error_page.404': self.error_page_404,
'request.error_response': self.error_page
})
csp = (
secure.ContentSecurityPolicy()
.default_src("'self'")
.script_src("'self'", "'unsafe-inline'", "blob:")
.style_src("'self'", "'unsafe-inline'")
.base_uri("'self'")
.connect_src("'self'", "data:")
.frame_src("'self'", 'data:')
.img_src("'self'", "data:")
)
secure_headers = secure.Secure(
server=secure.Server().set("server"),
cache=secure.CacheControl().must_revalidate(),
csp=csp,
referrer=secure.ReferrerPolicy().no_referrer(),
)
cherrypy.config.update({
"tools.response_headers.on": True,
"tools.response_headers.headers": secure_headers.framework.cherrypy()
})
def __init__(self, config):
self.defaultConfig = deepcopy(config)
dbh = SpiderFootDb(config)
# 'config' supplied will be the defaults, let's supplement them
# now with any configuration which may have previously been
# saved.
sf = SpiderFoot(config)
self.config = sf.configUnserialize(dbh.configGet(), config)
def test_config_unserialize_should_return_a_dict(self):
"""
Test configUnserialize(self, opts, referencePoint, filterSystem=True)
"""
sf = SpiderFoot(dict())
config = sf.configUnserialize(dict(), dict(), True)
self.assertIsInstance(config, dict)
def __init__(self, config):
"""Initialize web server
Args:
config: TBD
"""
if not isinstance(config, dict):
raise TypeError("config is %s; expected dict()" % type(config))
if not config:
raise ValueError("config is empty")
self.defaultConfig = deepcopy(config)
dbh = SpiderFootDb(self.defaultConfig)
# 'config' supplied will be the defaults, let's supplement them
# now with any configuration which may have previously been
# saved.
sf = SpiderFoot(self.defaultConfig)
self.config = sf.configUnserialize(dbh.configGet(), self.defaultConfig)
if self.config['__webaddr'] == "0.0.0.0": # nosec
addr = "<IP of this host>"
else:
addr = self.config['__webaddr']
self.docroot = self.config['__docroot'].rstrip('/')
cherrypy.config.update({
'error_page.404': self.error_page_404,
'request.error_response': self.error_page
})
secure_headers = SecureHeaders(
server="server",
cache=False,
csp="default-src 'self' ; script-src 'self' 'unsafe-inline' blob: ; style-src 'self' 'unsafe-inline' ; img-src 'self' data:"
)
cherrypy.config.update({
"tools.response_headers.on": True,
"tools.response_headers.headers": secure_headers.cherrypy()
})
if (cherrypy.server.ssl_certificate is None or cherrypy.server.ssl_private_key is None):
url = "http://%s:%s%s" % (addr, self.config['__webport'], self.docroot)
else:
url = "https://%s:%s%s" % (addr, self.config['__webport'], self.docroot)
print("")
print("")
print("*************************************************************")
print(" Use SpiderFoot by starting your web browser of choice and ")
print(" browse to %s" % url)
print("*************************************************************")
print("")
print("")
def savesettings(self, allopts, token, configFile=None):
if str(token) != str(self.token):
return self.error("Invalid token (" + str(self.token) + ").")
if configFile: # configFile seems to get set even if a file isn't uploaded
if configFile.file:
contents = configFile.file.read()
if type(contents) == bytes:
contents = contents.decode("utf-8")
try:
tmp = dict()
for line in contents.split("\n"):
if "=" not in line:
continue
l = line.strip().split("=")
if len(l) == 1:
l[1] = ""
tmp[l[0]] = l[1]
allopts = json.dumps(tmp)
except BaseException as e:
return self.error(
"Failed to parse input file. Was it generated from SpiderFoot? ("
+ str(e) + ")")
try:
dbh = SpiderFootDb(self.config)
# Reset config to default
if allopts == "RESET":
dbh.configClear() # Clear it in the DB
self.config = deepcopy(self.defaultConfig) # Clear in memory
else:
useropts = json.loads(allopts)
cleanopts = dict()
for opt in list(useropts.keys()):
cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0]
currentopts = deepcopy(self.config)
# Make a new config where the user options override
# the current system config.
sf = SpiderFoot(self.config)
self.config = sf.configUnserialize(cleanopts, currentopts)
dbh.configSet(sf.configSerialize(self.config))
except Exception as e:
return self.error(
"Processing one or more of your inputs failed: " + str(e))
templ = Template(filename='dyn/opts.tmpl', lookup=self.lookup)
self.token = random.SystemRandom().randint(0, 99999999)
return templ.render(opts=self.config,
pageid='SETTINGS',
updated=True,
docroot=self.docroot,
token=self.token)
def __init__(self, config):
if not isinstance(config, dict):
raise TypeError("config is %s; expected dict()" % type(config))
if not config:
raise ValueError("config is empty")
self.defaultConfig = deepcopy(config)
dbh = SpiderFootDb(self.defaultConfig)
# 'config' supplied will be the defaults, let's supplement them
# now with any configuration which may have previously been
# saved.
sf = SpiderFoot(self.defaultConfig)
self.config = sf.configUnserialize(dbh.configGet(), self.defaultConfig)
if self.config['__webaddr'] == "0.0.0.0": # nosec
addr = "<IP of this host>"
else:
addr = self.config['__webaddr']
self.docroot = self.config['__docroot'].rstrip('/')
cherrypy.config.update({
'error_page.404': self.error_page_404,
'request.error_response': self.error_page
})
secure_headers = SecureHeaders(
csp=
"default-src 'self'; script-src 'self' 'unsafe-inline' blob: ; style-src 'self' 'unsafe-inline'"
)
cherrypy.config.update({
"tools.response_headers.on":
True,
"tools.response_headers.headers":
secure_headers.cherrypy()
})
print("")
print("")
print("*************************************************************")
print(" Use SpiderFoot by starting your web browser of choice and ")
print(" browse to http://" + addr + ":" +
str(self.config['__webport']) + self.docroot)
print("*************************************************************")
print("")
print("")
def savesettings(self, allopts, token, configFile=None):
if str(token) != str(self.token):
return self.error("Invalid token (" + str(self.token) + ").")
if configFile: # configFile seems to get set even if a file isn't uploaded
if configFile.file:
contents = configFile.file.read()
try:
tmp = dict()
for line in contents.split("\n"):
if "=" not in line:
continue
l = line.strip().split("=")
if len(l) == 1:
l[1] = ""
tmp[l[0]] = l[1]
allopts = json.dumps(tmp)
except BaseException as e:
return self.error("Failed to parse input file. Was it generated from SpiderFoot? (" + str(e) + ")")
try:
dbh = SpiderFootDb(self.config)
# Reset config to default
if allopts == "RESET":
dbh.configClear() # Clear it in the DB
self.config = deepcopy(self.defaultConfig) # Clear in memory
else:
useropts = json.loads(allopts)
cleanopts = dict()
for opt in useropts.keys():
cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0]
currentopts = deepcopy(self.config)
# Make a new config where the user options override
# the current system config.
sf = SpiderFoot(self.config)
self.config = sf.configUnserialize(cleanopts, currentopts)
dbh.configSet(sf.configSerialize(self.config))
except Exception as e:
return self.error("Processing one or more of your inputs failed: " + str(e))
templ = Template(filename='dyn/opts.tmpl', lookup=self.lookup)
self.token = random.randint(0, 99999999)
return templ.render(opts=self.config, pageid='SETTINGS', updated=True,
docroot=self.docroot, token=self.token)
def __init__(self, web_config, config):
"""Initialize web server
Args:
web_config: config settings for web interface (interface, port, root path)
config: SpiderFoot config
"""
if not isinstance(config, dict):
raise TypeError(f"config is {type(config)}; expected dict()")
if not config:
raise ValueError("config is empty")
if not isinstance(web_config, dict):
raise TypeError(f"web_config is {type(web_config)}; expected dict()")
if not config:
raise ValueError("web_config is empty")
self.docroot = web_config.get('root', '/').rstrip('/')
# 'config' supplied will be the defaults, let's supplement them
# now with any configuration which may have previously been saved.
self.defaultConfig = deepcopy(config)
dbh = SpiderFootDb(self.defaultConfig)
sf = SpiderFoot(self.defaultConfig)
self.config = sf.configUnserialize(dbh.configGet(), self.defaultConfig)
cherrypy.config.update({
'error_page.404': self.error_page_404,
'request.error_response': self.error_page
})
secure_headers = SecureHeaders(
server="server",
cache=False,
csp="default-src 'self' ; script-src 'self' 'unsafe-inline' blob: ; style-src 'self' 'unsafe-inline' ; img-src 'self' data:"
)
cherrypy.config.update({
"tools.response_headers.on": True,
"tools.response_headers.headers": secure_headers.cherrypy()
})
def savesettingsraw(self, allopts, token):
"""Save settings, also used to completely reset them to default
Args:
allopts: TBD
token: TBD
"""
if str(token) != str(self.token):
return json.dumps(
["ERROR", "Invalid token (" + str(self.token) + ")."])
# Reset config to default
if allopts == "RESET":
if self.reset_settings():
return json.dumps(["SUCCESS", ""])
else:
return json.dumps(["ERROR", "Failed to reset settings"])
# Save settings
try:
dbh = SpiderFootDb(self.config)
useropts = json.loads(allopts)
cleanopts = dict()
for opt in list(useropts.keys()):
cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0]
currentopts = deepcopy(self.config)
# Make a new config where the user options override
# the current system config.
sf = SpiderFoot(self.config)
self.config = sf.configUnserialize(cleanopts, currentopts)
dbh.configSet(sf.configSerialize(self.config))
except Exception as e:
return json.dumps([
"ERROR",
"Processing one or more of your inputs failed: %s" % e
])
return json.dumps(["SUCCESS", ""])
def savesettingsraw(self, allopts, token):
"""Save settings, also used to completely reset them to default
Args:
allopts: TBD
token: CSRF token
Returns:
str: save success as JSON
"""
cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8"
if str(token) != str(self.token):
return json.dumps(["ERROR", "Invalid token (%s)." % self.token]).encode('utf-8')
# Reset config to default
if allopts == "RESET":
if self.reset_settings():
return json.dumps(["SUCCESS", ""]).encode('utf-8')
return json.dumps(["ERROR", "Failed to reset settings"]).encode('utf-8')
# Save settings
try:
dbh = SpiderFootDb(self.config)
useropts = json.loads(allopts)
cleanopts = dict()
for opt in list(useropts.keys()):
cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0]
currentopts = deepcopy(self.config)
# Make a new config where the user options override
# the current system config.
sf = SpiderFoot(self.config)
self.config = sf.configUnserialize(cleanopts, currentopts)
dbh.configSet(sf.configSerialize(self.config))
except Exception as e:
return json.dumps(["ERROR", "Processing one or more of your inputs failed: %s" % e]).encode('utf-8')
return json.dumps(["SUCCESS", ""]).encode('utf-8')
def __init__(self, config):
self.defaultConfig = deepcopy(config)
dbh = SpiderFootDb(config)
# 'config' supplied will be the defaults, let's supplement them
# now with any configuration which may have previously been
# saved.
sf = SpiderFoot(config)
self.config = sf.configUnserialize(dbh.configGet(), config)
if self.config['__webaddr'] == "0.0.0.0":
addr = "<IP of this host>"
else:
addr = self.config['__webaddr']
print ""
print ""
print "*************************************************************"
print " Use SpiderFoot by starting your web browser of choice and "
print " browse to http://" + addr + ":" + str(self.config['__webport'])
print "*************************************************************"
print ""
print ""
def savesettings(self, allopts):
try:
dbh = SpiderFootDb(self.config)
# Reset config to default
if allopts == "RESET":
dbh.configClear() # Clear it in the DB
self.config = deepcopy(self.defaultConfig) # Clear in memory
else:
useropts = json.loads(allopts)
currentopts = deepcopy(self.config)
# Make a new config where the user options override
# the current system config.
sf = SpiderFoot(self.config)
self.config = sf.configUnserialize(useropts, currentopts)
dbh.configSet(sf.configSerialize(currentopts))
except Exception as e:
return self.error("Processing one or more of your inputs failed: " + str(e))
templ = Template(filename='dyn/opts.tmpl', lookup=self.lookup)
return templ.render(opts=self.config, pageid='SETTINGS', updated=True)
def savesettings(self, allopts, token, configFile=None):
"""Save settings, also used to completely reset them to default
Args:
allopts: TBD
token: TBD
configFile: TBD
"""
if str(token) != str(self.token):
return self.error("Invalid token (" + str(self.token) + ").")
if configFile: # configFile seems to get set even if a file isn't uploaded
if configFile.file:
contents = configFile.file.read()
if type(contents) == bytes:
contents = contents.decode("utf-8")
try:
tmp = dict()
for line in contents.split("\n"):
if "=" not in line:
continue
opt_array = line.strip().split("=")
if len(opt_array) == 1:
opt_array[1] = ""
tmp[opt_array[0]] = '='.join(opt_array[1:])
allopts = json.dumps(tmp)
except BaseException as e:
return self.error(
"Failed to parse input file. Was it generated from SpiderFoot? (%s)"
% e)
# Reset config to default
if allopts == "RESET":
if self.reset_settings():
raise cherrypy.HTTPRedirect("/opts?updated=1")
else:
return self.error("Failed to reset settings")
# Save settings
try:
dbh = SpiderFootDb(self.config)
useropts = json.loads(allopts)
cleanopts = dict()
for opt in list(useropts.keys()):
cleanopts[opt] = self.cleanUserInput([useropts[opt]])[0]
currentopts = deepcopy(self.config)
# Make a new config where the user options override
# the current system config.
sf = SpiderFoot(self.config)
self.config = sf.configUnserialize(cleanopts, currentopts)
dbh.configSet(sf.configSerialize(self.config))
except Exception as e:
return self.error(
"Processing one or more of your inputs failed: %s" % e)
raise cherrypy.HTTPRedirect("/opts?updated=1")
continue
if r in sfModules[mod].get('provides',
[]) and mod not in modlist:
modlist.append(mod)
if len(modlist) == 0:
print("Based on your criteria, no modules were enabled.")
sys.exit(-1)
modlist += ["sfp__stor_db", "sfp__stor_stdout"]
# Run the scan
if sfConfig['__logging']:
print(("[*] Modules enabled (" + str(len(modlist)) + "): " +
",".join(modlist)))
cfg = sf.configUnserialize(dbh.configGet(), sfConfig)
scanId = sf.genScanInstanceGUID(target)
# Debug mode is a variable that gets stored to the DB, so re-apply it
if args.debug:
cfg['_debug'] = True
else:
cfg['_debug'] = False
# If strict mode is enabled, filter the output from modules.
if args.x and args.t:
cfg['__outputfilter'] = args.t.split(",")
t = SpiderFootScanner(target, target, targetType, scanId, modlist, cfg,
dict())
t.daemon = True
def start_scan(sfConfig: dict, sfModules: dict, args, loggingQueue) -> None:
"""Start scan
Args:
sfConfig (dict): SpiderFoot config options
sfModules (dict): modules
args (argparse.Namespace): command line args
loggingQueue (Queue): main SpiderFoot logging queue
"""
log = logging.getLogger(f"spiderfoot.{__name__}")
global dbh
global scanId
dbh = SpiderFootDb(sfConfig, init=True)
sf = SpiderFoot(sfConfig)
if not args.s:
log.error(
"You must specify a target when running in scan mode. Try --help for guidance."
)
sys.exit(-1)
if args.x and not args.t:
log.error("-x can only be used with -t. Use --help for guidance.")
sys.exit(-1)
if args.x and args.m:
log.error(
"-x can only be used with -t and not with -m. Use --help for guidance."
)
sys.exit(-1)
if args.r and (args.o and args.o not in ["tab", "csv"]):
log.error("-r can only be used when your output format is tab or csv.")
sys.exit(-1)
if args.H and (args.o and args.o not in ["tab", "csv"]):
log.error("-H can only be used when your output format is tab or csv.")
sys.exit(-1)
if args.D and args.o != "csv":
log.error("-D can only be used when using the csv output format.")
sys.exit(-1)
target = args.s
# Usernames and names - quoted on the commandline - won't have quotes,
# so add them.
if " " in target:
target = f"\"{target}\""
if "." not in target and not target.startswith("+") and '"' not in target:
target = f"\"{target}\""
targetType = SpiderFootHelpers.targetTypeFromString(target)
if not targetType:
log.error(f"Could not determine target type. Invalid target: {target}")
sys.exit(-1)
target = target.strip('"')
modlist = list()
if not args.t and not args.m and not args.u:
log.warning(
"You didn't specify any modules, types or use case, so all modules will be enabled."
)
for m in list(sfModules.keys()):
if "__" in m:
continue
modlist.append(m)
signal.signal(signal.SIGINT, handle_abort)
# If the user is scanning by type..
# 1. Find modules producing that type
if args.t:
types = args.t
modlist = sf.modulesProducing(types)
newmods = deepcopy(modlist)
newmodcpy = deepcopy(newmods)
# 2. For each type those modules consume, get modules producing
while len(newmodcpy) > 0:
for etype in sf.eventsToModules(newmodcpy):
xmods = sf.modulesProducing([etype])
for mod in xmods:
if mod not in modlist:
modlist.append(mod)
newmods.append(mod)
newmodcpy = deepcopy(newmods)
newmods = list()
# Easier if scanning by module
if args.m:
modlist = list(filter(None, args.m.split(",")))
# Select modules if the user selected usercase
if args.u:
usecase = args.u[0].upper() + args.u[
1:] # Make the first Letter Uppercase
for mod in sfConfig['__modules__']:
if usecase == 'All' or usecase in sfConfig['__modules__'][mod][
'group']:
modlist.append(mod)
# Add sfp__stor_stdout to the module list
typedata = dbh.eventTypes()
types = dict()
for r in typedata:
types[r[1]] = r[0]
sfp__stor_stdout_opts = sfConfig['__modules__']['sfp__stor_stdout']['opts']
sfp__stor_stdout_opts['_eventtypes'] = types
if args.f:
if args.f and not args.t:
log.error("You can only use -f with -t. Use --help for guidance.")
sys.exit(-1)
sfp__stor_stdout_opts['_showonlyrequested'] = True
if args.F:
sfp__stor_stdout_opts['_requested'] = args.F.split(",")
sfp__stor_stdout_opts['_showonlyrequested'] = True
if args.o:
if args.o not in ["tab", "csv", "json"]:
log.error(
"Invalid output format selected. Must be 'tab', 'csv' or 'json'."
)
sys.exit(-1)
sfp__stor_stdout_opts['_format'] = args.o
if args.t:
sfp__stor_stdout_opts['_requested'] = args.t.split(",")
if args.n:
sfp__stor_stdout_opts['_stripnewline'] = True
if args.r:
sfp__stor_stdout_opts['_showsource'] = True
if args.S:
sfp__stor_stdout_opts['_maxlength'] = args.S
if args.D:
sfp__stor_stdout_opts['_csvdelim'] = args.D
if args.x:
tmodlist = list()
modlist = list()
xmods = sf.modulesConsuming([targetType])
for mod in xmods:
if mod not in modlist:
tmodlist.append(mod)
# Remove any modules not producing the type requested
rtypes = args.t.split(",")
for mod in tmodlist:
for r in rtypes:
if not sfModules[mod]['provides']:
continue
if r in sfModules[mod].get('provides',
[]) and mod not in modlist:
modlist.append(mod)
if len(modlist) == 0:
log.error("Based on your criteria, no modules were enabled.")
sys.exit(-1)
modlist += ["sfp__stor_db", "sfp__stor_stdout"]
if sfConfig['__logging']:
log.info(f"Modules enabled ({len(modlist)}): {','.join(modlist)}")
cfg = sf.configUnserialize(dbh.configGet(), sfConfig)
# Debug mode is a variable that gets stored to the DB, so re-apply it
if args.debug:
cfg['_debug'] = True
else:
cfg['_debug'] = False
# If strict mode is enabled, filter the output from modules.
if args.x and args.t:
cfg['__outputfilter'] = args.t.split(",")
# Prepare scan output headers
if args.o == "json":
print("[", end='')
elif not args.H:
delim = "\t"
if args.o == "tab":
delim = "\t"
if args.o == "csv":
if args.D:
delim = args.D
else:
delim = ","
if args.r:
if delim == "\t":
headers = '{0:30}{1}{2:45}{3}{4}{5}{6}'.format(
"Source", delim, "Type", delim, "Source Data", delim,
"Data")
else:
headers = delim.join(["Source", "Type", "Source Data", "Data"])
else:
if delim == "\t":
headers = '{0:30}{1}{2:45}{3}{4}'.format(
"Source", delim, "Type", delim, "Data")
else:
headers = delim.join(["Source", "Type", "Data"])
print(headers)
# Start running a new scan
scanName = target
scanId = SpiderFootHelpers.genScanInstanceId()
try:
p = mp.Process(target=startSpiderFootScanner,
args=(loggingQueue, scanName, scanId, target,
targetType, modlist, cfg))
p.daemon = True
p.start()
except BaseException as e:
log.error(f"Scan [{scanId}] failed: {e}")
sys.exit(-1)
# Poll for scan status until completion
while True:
time.sleep(1)
info = dbh.scanInstanceGet(scanId)
if not info:
continue
if info[5] in [
"ERROR-FAILED", "ABORT-REQUESTED", "ABORTED", "FINISHED"
]:
if sfConfig['__logging']:
log.info(f"Scan completed with status {info[5]}")
if args.o == "json":
print("]")
sys.exit(0)
return
def start_scan(sfConfig, sfModules, args):
global dbh
global scanId
dbh = SpiderFootDb(sfConfig, init=True)
sf = SpiderFoot(sfConfig)
if args.modules:
log.info("Modules available:")
for m in sorted(sfModules.keys()):
if "__" in m:
continue
print(('{0:25} {1}'.format(m, sfModules[m]['descr'])))
sys.exit(0)
if args.types:
log.info("Types available:")
typedata = dbh.eventTypes()
types = dict()
for r in typedata:
types[r[1]] = r[0]
for t in sorted(types.keys()):
print(('{0:45} {1}'.format(t, types[t])))
sys.exit(0)
if not args.s:
log.error(
"You must specify a target when running in scan mode. Try --help for guidance."
)
sys.exit(-1)
if args.x and not args.t:
log.error("-x can only be used with -t. Use --help for guidance.")
sys.exit(-1)
if args.x and args.m:
log.error(
"-x can only be used with -t and not with -m. Use --help for guidance."
)
sys.exit(-1)
if args.r and (args.o and args.o not in ["tab", "csv"]):
log.error("-r can only be used when your output format is tab or csv.")
sys.exit(-1)
if args.H and (args.o and args.o not in ["tab", "csv"]):
log.error("-H can only be used when your output format is tab or csv.")
sys.exit(-1)
if args.D and args.o != "csv":
log.error("-D can only be used when using the csv output format.")
sys.exit(-1)
target = args.s
# Usernames and names - quoted on the commandline - won't have quotes,
# so add them.
if " " in target:
target = f"\"{target}\""
if "." not in target and not target.startswith("+") and '"' not in target:
target = f"\"{target}\""
targetType = sf.targetType(target)
if not targetType:
log.error(f"Could not determine target type. Invalid target: {target}")
sys.exit(-1)
target = target.strip('"')
modlist = list()
if not args.t and not args.m:
log.warning(
"You didn't specify any modules or types, so all will be enabled.")
for m in list(sfModules.keys()):
if "__" in m:
continue
modlist.append(m)
signal.signal(signal.SIGINT, handle_abort)
# If the user is scanning by type..
# 1. Find modules producing that type
if args.t:
types = args.t
modlist = sf.modulesProducing(types)
newmods = deepcopy(modlist)
newmodcpy = deepcopy(newmods)
# 2. For each type those modules consume, get modules producing
while len(newmodcpy) > 0:
for etype in sf.eventsToModules(newmodcpy):
xmods = sf.modulesProducing([etype])
for mod in xmods:
if mod not in modlist:
modlist.append(mod)
newmods.append(mod)
newmodcpy = deepcopy(newmods)
newmods = list()
# Easier if scanning by module
if args.m:
modlist = list(filter(None, args.m.split(",")))
# Add sfp__stor_stdout to the module list
typedata = dbh.eventTypes()
types = dict()
for r in typedata:
types[r[1]] = r[0]
sfp__stor_stdout_opts = sfConfig['__modules__']['sfp__stor_stdout']['opts']
sfp__stor_stdout_opts['_eventtypes'] = types
if args.f:
if args.f and not args.t:
log.error("You can only use -f with -t. Use --help for guidance.")
sys.exit(-1)
sfp__stor_stdout_opts['_showonlyrequested'] = True
if args.F:
sfp__stor_stdout_opts['_requested'] = args.F.split(",")
sfp__stor_stdout_opts['_showonlyrequested'] = True
if args.o:
sfp__stor_stdout_opts['_format'] = args.o
if args.t:
sfp__stor_stdout_opts['_requested'] = args.t.split(",")
if args.n:
sfp__stor_stdout_opts['_stripnewline'] = True
if args.r:
sfp__stor_stdout_opts['_showsource'] = True
if args.S:
sfp__stor_stdout_opts['_maxlength'] = args.S
if args.D:
sfp__stor_stdout_opts['_csvdelim'] = args.D
if args.x:
tmodlist = list()
modlist = list()
xmods = sf.modulesConsuming([targetType])
for mod in xmods:
if mod not in modlist:
tmodlist.append(mod)
# Remove any modules not producing the type requested
rtypes = args.t.split(",")
for mod in tmodlist:
for r in rtypes:
if not sfModules[mod]['provides']:
continue
if r in sfModules[mod].get('provides',
[]) and mod not in modlist:
modlist.append(mod)
if len(modlist) == 0:
log.error("Based on your criteria, no modules were enabled.")
sys.exit(-1)
modlist += ["sfp__stor_db", "sfp__stor_stdout"]
# Run the scan
if sfConfig['__logging']:
log.info(f"Modules enabled ({len(modlist)}): {','.join(modlist)}")
cfg = sf.configUnserialize(dbh.configGet(), sfConfig)
# Debug mode is a variable that gets stored to the DB, so re-apply it
if args.debug:
cfg['_debug'] = True
else:
cfg['_debug'] = False
# If strict mode is enabled, filter the output from modules.
if args.x and args.t:
cfg['__outputfilter'] = args.t.split(",")
if args.o == "json":
print("[", end='')
# Start running a new scan
scanName = target
scanId = sf.genScanInstanceId()
try:
p = mp.Process(target=SpiderFootScanner,
args=(scanName, scanId, target, targetType, modlist,
cfg))
p.daemon = True
p.start()
except BaseException as e:
log.error(f"Scan [{scanId}] failed: {e}")
sys.exit(-1)
# If field headers weren't disabled, print them
if not args.H and args.o != "json":
if args.D:
delim = args.D
else:
if args.o in ["tab", None]:
delim = "\t"
if args.o == "csv":
delim = ","
if not args.r:
if delim != "\t":
print(delim.join(["Source", "Type", "Data"]))
else:
print('{0:30}{1}{2:45}{3}{4}'.format("Source", delim, "Type",
delim, "Data"))
else:
if delim != "\t":
print(delim.join(["Source", "Type", "Source Data", "Data"]))
else:
print('{0:30}{1}{2:45}{3}{4}{5}{6}'.format(
"Source", delim, "Type", delim, "Source Data", delim,
"Data"))
while True:
time.sleep(1)
info = dbh.scanInstanceGet(scanId)
if not info:
continue
if info[5] in [
"ERROR-FAILED", "ABORT-REQUESTED", "ABORTED", "FINISHED"
]:
if sfConfig['__logging']:
log.info(f"Scan completed with status {info[5]}")
if args.o == "json":
print("]")
sys.exit(0)
return None
for r in rtypes:
if not sfModules[mod]['provides']:
continue
if r in sfModules[mod].get('provides', []) and mod not in modlist:
modlist.append(mod)
if len(modlist) == 0:
print "Based on your criteria, no modules were enabled."
sys.exit(-1)
modlist += ["sfp__stor_db", "sfp__stor_stdout"]
# Run the scan
if not args.q:
print "[*] Modules enabled (" + str(len(modlist)) + "): " + ",".join(modlist)
cfg = sf.configUnserialize(dbh.configGet(), sfConfig)
scanId = sf.genScanInstanceGUID(target)
# Debug mode is a variable that gets stored to the DB, so re-apply it
if args.debug:
cfg['_debug'] = True
else:
cfg['_debug'] = False
# If strict mode is enabled, filter the output from modules.
if args.x and args.t:
cfg['__outputfilter'] = args.t.split(",")
t = SpiderFootScanner(target, target, targetType, scanId,
modlist, cfg, dict())
t.daemon = True
