def test_configGet_should_return_a_dict(self): """ Test configGet(self) """ sfdb = SpiderFootDb(self.default_options, False) config = sfdb.configGet() self.assertIsInstance(config, dict)
def __init__(self, web_config, config): """Initialize web server Args: web_config: config settings for web interface (interface, port, root path) config: SpiderFoot config Raises: TypeError: arg type is invalid ValueError: arg value is invalid """ if not isinstance(config, dict): raise TypeError(f"config is {type(config)}; expected dict()") if not config: raise ValueError("config is empty") if not isinstance(web_config, dict): raise TypeError(f"web_config is {type(web_config)}; expected dict()") if not config: raise ValueError("web_config is empty") self.docroot = web_config.get('root', '/').rstrip('/') # 'config' supplied will be the defaults, let's supplement them # now with any configuration which may have previously been saved. self.defaultConfig = deepcopy(config) dbh = SpiderFootDb(self.defaultConfig) sf = SpiderFoot(self.defaultConfig) self.config = sf.configUnserialize(dbh.configGet(), self.defaultConfig) cherrypy.config.update({ 'error_page.401': self.error_page_401, 'error_page.404': self.error_page_404, 'request.error_response': self.error_page }) csp = ( secure.ContentSecurityPolicy() .default_src("'self'") .script_src("'self'", "'unsafe-inline'", "blob:") .style_src("'self'", "'unsafe-inline'") .base_uri("'self'") .connect_src("'self'", "data:") .frame_src("'self'", 'data:') .img_src("'self'", "data:") ) secure_headers = secure.Secure( server=secure.Server().set("server"), cache=secure.CacheControl().must_revalidate(), csp=csp, referrer=secure.ReferrerPolicy().no_referrer(), ) cherrypy.config.update({ "tools.response_headers.on": True, "tools.response_headers.headers": secure_headers.framework.cherrypy() })
def __init__(self, config): """Initialize web server Args: config: TBD """ if not isinstance(config, dict): raise TypeError("config is %s; expected dict()" % type(config)) if not config: raise ValueError("config is empty") self.defaultConfig = deepcopy(config) dbh = SpiderFootDb(self.defaultConfig) # 'config' supplied will be the defaults, let's supplement them # now with any configuration which may have previously been # saved. sf = SpiderFoot(self.defaultConfig) self.config = sf.configUnserialize(dbh.configGet(), self.defaultConfig) if self.config['__webaddr'] == "0.0.0.0": # nosec addr = "<IP of this host>" else: addr = self.config['__webaddr'] self.docroot = self.config['__docroot'].rstrip('/') cherrypy.config.update({ 'error_page.404': self.error_page_404, 'request.error_response': self.error_page }) secure_headers = SecureHeaders( server="server", cache=False, csp="default-src 'self' ; script-src 'self' 'unsafe-inline' blob: ; style-src 'self' 'unsafe-inline' ; img-src 'self' data:" ) cherrypy.config.update({ "tools.response_headers.on": True, "tools.response_headers.headers": secure_headers.cherrypy() }) if (cherrypy.server.ssl_certificate is None or cherrypy.server.ssl_private_key is None): url = "http://%s:%s%s" % (addr, self.config['__webport'], self.docroot) else: url = "https://%s:%s%s" % (addr, self.config['__webport'], self.docroot) print("") print("") print("*************************************************************") print(" Use SpiderFoot by starting your web browser of choice and ") print(" browse to %s" % url) print("*************************************************************") print("") print("")
def test_configClear_should_clear_config(self): """ Test configClear(self) """ sfdb = SpiderFootDb(self.default_options, False) opts = dict() opts['example'] = 'example non-default config opt' sfdb.configSet(opts) config = sfdb.configGet() self.assertIsInstance(config, dict) self.assertIn('example', config) sfdb.configClear() config = sfdb.configGet() self.assertIsInstance(config, dict) self.assertNotIn('example', config)
def test_configSet_should_set_config_opts(self): """ Test configSet(self, optMap=dict()) """ sfdb = SpiderFootDb(self.default_options, False) opts = dict() opts['example'] = 'example non-default config opt' sfdb.configSet(opts) config = sfdb.configGet() self.assertIsInstance(config, dict) self.assertIn('example', config) self.assertEqual('TBD', 'TBD')
def __init__(self, web_config, config): """Initialize web server Args: web_config: config settings for web interface (interface, port, root path) config: SpiderFoot config """ if not isinstance(config, dict): raise TypeError(f"config is {type(config)}; expected dict()") if not config: raise ValueError("config is empty") if not isinstance(web_config, dict): raise TypeError(f"web_config is {type(web_config)}; expected dict()") if not config: raise ValueError("web_config is empty") self.docroot = web_config.get('root', '/').rstrip('/') # 'config' supplied will be the defaults, let's supplement them # now with any configuration which may have previously been saved. self.defaultConfig = deepcopy(config) dbh = SpiderFootDb(self.defaultConfig) sf = SpiderFoot(self.defaultConfig) self.config = sf.configUnserialize(dbh.configGet(), self.defaultConfig) cherrypy.config.update({ 'error_page.404': self.error_page_404, 'request.error_response': self.error_page }) secure_headers = SecureHeaders( server="server", cache=False, csp="default-src 'self' ; script-src 'self' 'unsafe-inline' blob: ; style-src 'self' 'unsafe-inline' ; img-src 'self' data:" ) cherrypy.config.update({ "tools.response_headers.on": True, "tools.response_headers.headers": secure_headers.cherrypy() })
def start_scan(sfConfig, sfModules, args): global dbh global scanId dbh = SpiderFootDb(sfConfig, init=True) sf = SpiderFoot(sfConfig) if args.modules: log.info("Modules available:") for m in sorted(sfModules.keys()): if "__" in m: continue print(('{0:25} {1}'.format(m, sfModules[m]['descr']))) sys.exit(0) if args.types: log.info("Types available:") typedata = dbh.eventTypes() types = dict() for r in typedata: types[r[1]] = r[0] for t in sorted(types.keys()): print(('{0:45} {1}'.format(t, types[t]))) sys.exit(0) if not args.s: log.error( "You must specify a target when running in scan mode. Try --help for guidance." ) sys.exit(-1) if args.x and not args.t: log.error("-x can only be used with -t. Use --help for guidance.") sys.exit(-1) if args.x and args.m: log.error( "-x can only be used with -t and not with -m. Use --help for guidance." ) sys.exit(-1) if args.r and (args.o and args.o not in ["tab", "csv"]): log.error("-r can only be used when your output format is tab or csv.") sys.exit(-1) if args.H and (args.o and args.o not in ["tab", "csv"]): log.error("-H can only be used when your output format is tab or csv.") sys.exit(-1) if args.D and args.o != "csv": log.error("-D can only be used when using the csv output format.") sys.exit(-1) target = args.s # Usernames and names - quoted on the commandline - won't have quotes, # so add them. if " " in target: target = f"\"{target}\"" if "." not in target and not target.startswith("+") and '"' not in target: target = f"\"{target}\"" targetType = sf.targetType(target) if not targetType: log.error(f"Could not determine target type. Invalid target: {target}") sys.exit(-1) target = target.strip('"') modlist = list() if not args.t and not args.m: log.warning( "You didn't specify any modules or types, so all will be enabled.") for m in list(sfModules.keys()): if "__" in m: continue modlist.append(m) signal.signal(signal.SIGINT, handle_abort) # If the user is scanning by type.. # 1. Find modules producing that type if args.t: types = args.t modlist = sf.modulesProducing(types) newmods = deepcopy(modlist) newmodcpy = deepcopy(newmods) # 2. For each type those modules consume, get modules producing while len(newmodcpy) > 0: for etype in sf.eventsToModules(newmodcpy): xmods = sf.modulesProducing([etype]) for mod in xmods: if mod not in modlist: modlist.append(mod) newmods.append(mod) newmodcpy = deepcopy(newmods) newmods = list() # Easier if scanning by module if args.m: modlist = list(filter(None, args.m.split(","))) # Add sfp__stor_stdout to the module list typedata = dbh.eventTypes() types = dict() for r in typedata: types[r[1]] = r[0] sfp__stor_stdout_opts = sfConfig['__modules__']['sfp__stor_stdout']['opts'] sfp__stor_stdout_opts['_eventtypes'] = types if args.f: if args.f and not args.t: log.error("You can only use -f with -t. Use --help for guidance.") sys.exit(-1) sfp__stor_stdout_opts['_showonlyrequested'] = True if args.F: sfp__stor_stdout_opts['_requested'] = args.F.split(",") sfp__stor_stdout_opts['_showonlyrequested'] = True if args.o: sfp__stor_stdout_opts['_format'] = args.o if args.t: sfp__stor_stdout_opts['_requested'] = args.t.split(",") if args.n: sfp__stor_stdout_opts['_stripnewline'] = True if args.r: sfp__stor_stdout_opts['_showsource'] = True if args.S: sfp__stor_stdout_opts['_maxlength'] = args.S if args.D: sfp__stor_stdout_opts['_csvdelim'] = args.D if args.x: tmodlist = list() modlist = list() xmods = sf.modulesConsuming([targetType]) for mod in xmods: if mod not in modlist: tmodlist.append(mod) # Remove any modules not producing the type requested rtypes = args.t.split(",") for mod in tmodlist: for r in rtypes: if not sfModules[mod]['provides']: continue if r in sfModules[mod].get('provides', []) and mod not in modlist: modlist.append(mod) if len(modlist) == 0: log.error("Based on your criteria, no modules were enabled.") sys.exit(-1) modlist += ["sfp__stor_db", "sfp__stor_stdout"] # Run the scan if sfConfig['__logging']: log.info(f"Modules enabled ({len(modlist)}): {','.join(modlist)}") cfg = sf.configUnserialize(dbh.configGet(), sfConfig) # Debug mode is a variable that gets stored to the DB, so re-apply it if args.debug: cfg['_debug'] = True else: cfg['_debug'] = False # If strict mode is enabled, filter the output from modules. if args.x and args.t: cfg['__outputfilter'] = args.t.split(",") if args.o == "json": print("[", end='') # Start running a new scan scanName = target scanId = sf.genScanInstanceId() try: p = mp.Process(target=SpiderFootScanner, args=(scanName, scanId, target, targetType, modlist, cfg)) p.daemon = True p.start() except BaseException as e: log.error(f"Scan [{scanId}] failed: {e}") sys.exit(-1) # If field headers weren't disabled, print them if not args.H and args.o != "json": if args.D: delim = args.D else: if args.o in ["tab", None]: delim = "\t" if args.o == "csv": delim = "," if not args.r: if delim != "\t": print(delim.join(["Source", "Type", "Data"])) else: print('{0:30}{1}{2:45}{3}{4}'.format("Source", delim, "Type", delim, "Data")) else: if delim != "\t": print(delim.join(["Source", "Type", "Source Data", "Data"])) else: print('{0:30}{1}{2:45}{3}{4}{5}{6}'.format( "Source", delim, "Type", delim, "Source Data", delim, "Data")) while True: time.sleep(1) info = dbh.scanInstanceGet(scanId) if not info: continue if info[5] in [ "ERROR-FAILED", "ABORT-REQUESTED", "ABORTED", "FINISHED" ]: if sfConfig['__logging']: log.info(f"Scan completed with status {info[5]}") if args.o == "json": print("]") sys.exit(0) return None
def start_scan(sfConfig: dict, sfModules: dict, args, loggingQueue) -> None: """Start scan Args: sfConfig (dict): SpiderFoot config options sfModules (dict): modules args (argparse.Namespace): command line args loggingQueue (Queue): main SpiderFoot logging queue """ log = logging.getLogger(f"spiderfoot.{__name__}") global dbh global scanId dbh = SpiderFootDb(sfConfig, init=True) sf = SpiderFoot(sfConfig) if not args.s: log.error( "You must specify a target when running in scan mode. Try --help for guidance." ) sys.exit(-1) if args.x and not args.t: log.error("-x can only be used with -t. Use --help for guidance.") sys.exit(-1) if args.x and args.m: log.error( "-x can only be used with -t and not with -m. Use --help for guidance." ) sys.exit(-1) if args.r and (args.o and args.o not in ["tab", "csv"]): log.error("-r can only be used when your output format is tab or csv.") sys.exit(-1) if args.H and (args.o and args.o not in ["tab", "csv"]): log.error("-H can only be used when your output format is tab or csv.") sys.exit(-1) if args.D and args.o != "csv": log.error("-D can only be used when using the csv output format.") sys.exit(-1) target = args.s # Usernames and names - quoted on the commandline - won't have quotes, # so add them. if " " in target: target = f"\"{target}\"" if "." not in target and not target.startswith("+") and '"' not in target: target = f"\"{target}\"" targetType = SpiderFootHelpers.targetTypeFromString(target) if not targetType: log.error(f"Could not determine target type. Invalid target: {target}") sys.exit(-1) target = target.strip('"') modlist = list() if not args.t and not args.m and not args.u: log.warning( "You didn't specify any modules, types or use case, so all modules will be enabled." ) for m in list(sfModules.keys()): if "__" in m: continue modlist.append(m) signal.signal(signal.SIGINT, handle_abort) # If the user is scanning by type.. # 1. Find modules producing that type if args.t: types = args.t modlist = sf.modulesProducing(types) newmods = deepcopy(modlist) newmodcpy = deepcopy(newmods) # 2. For each type those modules consume, get modules producing while len(newmodcpy) > 0: for etype in sf.eventsToModules(newmodcpy): xmods = sf.modulesProducing([etype]) for mod in xmods: if mod not in modlist: modlist.append(mod) newmods.append(mod) newmodcpy = deepcopy(newmods) newmods = list() # Easier if scanning by module if args.m: modlist = list(filter(None, args.m.split(","))) # Select modules if the user selected usercase if args.u: usecase = args.u[0].upper() + args.u[ 1:] # Make the first Letter Uppercase for mod in sfConfig['__modules__']: if usecase == 'All' or usecase in sfConfig['__modules__'][mod][ 'group']: modlist.append(mod) # Add sfp__stor_stdout to the module list typedata = dbh.eventTypes() types = dict() for r in typedata: types[r[1]] = r[0] sfp__stor_stdout_opts = sfConfig['__modules__']['sfp__stor_stdout']['opts'] sfp__stor_stdout_opts['_eventtypes'] = types if args.f: if args.f and not args.t: log.error("You can only use -f with -t. Use --help for guidance.") sys.exit(-1) sfp__stor_stdout_opts['_showonlyrequested'] = True if args.F: sfp__stor_stdout_opts['_requested'] = args.F.split(",") sfp__stor_stdout_opts['_showonlyrequested'] = True if args.o: if args.o not in ["tab", "csv", "json"]: log.error( "Invalid output format selected. Must be 'tab', 'csv' or 'json'." ) sys.exit(-1) sfp__stor_stdout_opts['_format'] = args.o if args.t: sfp__stor_stdout_opts['_requested'] = args.t.split(",") if args.n: sfp__stor_stdout_opts['_stripnewline'] = True if args.r: sfp__stor_stdout_opts['_showsource'] = True if args.S: sfp__stor_stdout_opts['_maxlength'] = args.S if args.D: sfp__stor_stdout_opts['_csvdelim'] = args.D if args.x: tmodlist = list() modlist = list() xmods = sf.modulesConsuming([targetType]) for mod in xmods: if mod not in modlist: tmodlist.append(mod) # Remove any modules not producing the type requested rtypes = args.t.split(",") for mod in tmodlist: for r in rtypes: if not sfModules[mod]['provides']: continue if r in sfModules[mod].get('provides', []) and mod not in modlist: modlist.append(mod) if len(modlist) == 0: log.error("Based on your criteria, no modules were enabled.") sys.exit(-1) modlist += ["sfp__stor_db", "sfp__stor_stdout"] if sfConfig['__logging']: log.info(f"Modules enabled ({len(modlist)}): {','.join(modlist)}") cfg = sf.configUnserialize(dbh.configGet(), sfConfig) # Debug mode is a variable that gets stored to the DB, so re-apply it if args.debug: cfg['_debug'] = True else: cfg['_debug'] = False # If strict mode is enabled, filter the output from modules. if args.x and args.t: cfg['__outputfilter'] = args.t.split(",") # Prepare scan output headers if args.o == "json": print("[", end='') elif not args.H: delim = "\t" if args.o == "tab": delim = "\t" if args.o == "csv": if args.D: delim = args.D else: delim = "," if args.r: if delim == "\t": headers = '{0:30}{1}{2:45}{3}{4}{5}{6}'.format( "Source", delim, "Type", delim, "Source Data", delim, "Data") else: headers = delim.join(["Source", "Type", "Source Data", "Data"]) else: if delim == "\t": headers = '{0:30}{1}{2:45}{3}{4}'.format( "Source", delim, "Type", delim, "Data") else: headers = delim.join(["Source", "Type", "Data"]) print(headers) # Start running a new scan scanName = target scanId = SpiderFootHelpers.genScanInstanceId() try: p = mp.Process(target=startSpiderFootScanner, args=(loggingQueue, scanName, scanId, target, targetType, modlist, cfg)) p.daemon = True p.start() except BaseException as e: log.error(f"Scan [{scanId}] failed: {e}") sys.exit(-1) # Poll for scan status until completion while True: time.sleep(1) info = dbh.scanInstanceGet(scanId) if not info: continue if info[5] in [ "ERROR-FAILED", "ABORT-REQUESTED", "ABORTED", "FINISHED" ]: if sfConfig['__logging']: log.info(f"Scan completed with status {info[5]}") if args.o == "json": print("]") sys.exit(0) return
if not sfModules[mod]['provides']: continue if r in sfModules[mod].get('provides', []) and mod not in modlist: modlist.append(mod) if len(modlist) == 0: log.error("Based on your criteria, no modules were enabled.") sys.exit(-1) modlist += ["sfp__stor_db", "sfp__stor_stdout"] # Run the scan if sfConfig['__logging']: log.info(f"Modules enabled ({len(modlist)}): {','.join(modlist)}") cfg = sf.configUnserialize(dbh.configGet(), sfConfig) # Debug mode is a variable that gets stored to the DB, so re-apply it if args.debug: cfg['_debug'] = True else: cfg['_debug'] = False # If strict mode is enabled, filter the output from modules. if args.x and args.t: cfg['__outputfilter'] = args.t.split(",") if args.o == "json": print("[", end='') # Start running a new scan