def rerunscan(self, id): """Rerun a scan Args: id (str): scan ID """ # Snapshot the current configuration to be used by the scan cfg = deepcopy(self.config) modlist = list() sf = SpiderFoot(cfg) dbh = SpiderFootDb(cfg) info = dbh.scanInstanceGet(id) if not info: return self.error("Invalid scan ID.") scanconfig = dbh.scanConfigGet(id) scanname = info[0] scantarget = info[1] targetType = None if len(scanconfig) == 0: return self.error("Something went wrong internally.") modlist = scanconfig['_modulesenabled'].split(',') if "sfp__stor_stdout" in modlist: modlist.remove("sfp__stor_stdout") targetType = sf.targetType(scantarget) if targetType is None: # It must then be a name, as a re-run scan should always have a clean # target. Put quotes around the target value and try to determine the # target type again. targetType = sf.targetType(f'"{scantarget}"') if targetType != "HUMAN_NAME": scantarget = scantarget.lower() # Start running a new scan scanId = sf.genScanInstanceGUID() try: p = mp.Process(target=SpiderFootScanner, args=(scanname, scanId, scantarget, targetType, modlist, cfg)) p.daemon = True p.start() except BaseException as e: print("[-] Scan [%s] failed: %s" % (scanId, e)) return self.error("Scan [%s] failed: %s" % (scanId, e)) # Wait until the scan has initialized while dbh.scanInstanceGet(scanId) is None: print("[info] Waiting for the scan to initialize...") time.sleep(1) raise cherrypy.HTTPRedirect(f"scaninfo?id={scanId}", status=302)
def rerunscanmulti(self, ids): """Rerun scans Args: ids (str): comma separated list of scan IDs """ # Snapshot the current configuration to be used by the scan cfg = deepcopy(self.config) modlist = list() sf = SpiderFoot(cfg) dbh = SpiderFootDb(cfg) for id in ids.split(","): info = dbh.scanInstanceGet(id) scanconfig = dbh.scanConfigGet(id) scanname = info[0] scantarget = info[1] targetType = None if len(scanconfig) == 0: return self.error("Something went wrong internally.") modlist = scanconfig['_modulesenabled'].split(',') if "sfp__stor_stdout" in modlist: modlist.remove("sfp__stor_stdout") targetType = sf.targetType(scantarget) if targetType is None: # Should never be triggered for a re-run scan.. return self.error( "Invalid target type. Could not recognize it as a human name, IP address, IP subnet, ASN, domain name or host name." ) # Start running a new scan scanId = sf.genScanInstanceGUID() try: p = mp.Process(target=SpiderFootScanner, args=(scanname, scanId, scantarget, targetType, modlist, cfg)) p.daemon = True p.start() except BaseException as e: print("[-] Scan [%s] failed: %s" % (scanId, e)) return self.error("Scan [%s] failed: %s" % (scanId, e)) # Wait until the scan has initialized while dbh.scanInstanceGet(scanId) is None: print("[info] Waiting for the scan to initialize...") time.sleep(1) templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup) return templ.render(rerunscans=True, docroot=self.docroot, pageid="SCANLIST")
def rerunscan(self, id): # Snapshot the current configuration to be used by the scan cfg = deepcopy(self.config) modopts = dict() # Not used yet as module options are set globally modlist = list() sf = SpiderFoot(cfg) dbh = SpiderFootDb(cfg) info = dbh.scanInstanceGet(id) scanconfig = dbh.scanConfigGet(id) scanname = info[0] scantarget = info[1] targetType = None if len(scanconfig) == 0: return self.error("Something went wrong internally.") modlist = scanconfig['_modulesenabled'].split(',') if "sfp__stor_stdout" in modlist: modlist.remove("sfp__stor_stdout") targetType = sf.targetType(scantarget) if targetType is None: # It must then be a name, as a re-run scan should always have a clean # target. Put quotes around the target value and try to determine the # target type again. targetType = sf.targetType(f'"{scantarget}"') if targetType != "HUMAN_NAME": scantarget = scantarget.lower() # Start running a new scan newId = sf.genScanInstanceGUID(scanname) p = mp.Process(target=SpiderFootScanner, args=(scanname, scantarget, targetType, newId, modlist, cfg, modopts)) p.start() # Wait until the scan has initialized while dbh.scanInstanceGet(newId) == None: print("[info] Waiting for the scan to initialize...") time.sleep(1) templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup) return templ.render(id=newId, name=str(scanname), docroot=self.docroot, status=dbh.scanInstanceGet(newId), pageid="SCANLIST")
def scansummary(self, id, by): """Summary of scan results Args: id (str): scan ID by: TBD """ retdata = [] dbh = SpiderFootDb(self.config) try: scandata = dbh.scanResultSummary(id, by) except Exception: return json.dumps(retdata) try: statusdata = dbh.scanInstanceGet(id) except Exception: return json.dumps(retdata) for row in scandata: if row[0] == "ROOT": continue lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[2])) retdata.append( [row[0], row[1], lastseen, row[3], row[4], statusdata[5]]) return json.dumps(retdata)
def stopscanmulti(self, ids): """Stop a scan Args: ids (str): comma separated list of scan IDs Note: Unnecessary for now given that only one simultaneous scan is permitted """ dbh = SpiderFootDb(self.config) error = list() for id in ids.split(","): scaninfo = dbh.scanInstanceGet(id) if not scaninfo: return self.error("Invalid scan ID: %s" % id) scanname = str(scaninfo[0]) scanstatus = scaninfo[5] if scanstatus == "FINISHED": error.append( "Scan '%s' is in a finished state. <a href='/scandelete?id=%s&confirm=1'>Maybe you want to delete it instead?</a>" % (scanname, id)) continue if scanstatus == "ABORTED": error.append("Scan '" + scanname + "' is already aborted.") continue dbh.scanInstanceSet(id, status="ABORT-REQUESTED") raise cherrypy.HTTPRedirect("/")
def scaneventresultexportmulti(self, ids, dialect="excel"): dbh = SpiderFootDb(self.config) scaninfo = dict() data = list() for id in ids.split(','): scaninfo[id] = dbh.scanInstanceGet(id) data = data + dbh.scanResultEvent(id) fileobj = StringIO() parser = csv.writer(fileobj, dialect=dialect) parser.writerow([ "Scan Name", "Updated", "Type", "Module", "Source", "F/P", "Data" ]) for row in data: if row[4] == "ROOT": continue lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0])) datafield = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "") parser.writerow([ scaninfo[row[12]][0], lastseen, str(row[4]), str(row[3]), str(row[2]), row[13], datafield ]) cherrypy.response.headers[ 'Content-Disposition'] = "attachment; filename=SpiderFoot.csv" cherrypy.response.headers['Content-Type'] = "application/csv" cherrypy.response.headers['Pragma'] = "no-cache" return fileobj.getvalue()
def clonescan(self, id): sf = SpiderFoot(self.config) dbh = SpiderFootDb(self.config) types = dbh.eventTypes() info = dbh.scanInstanceGet(id) scanconfig = dbh.scanConfigGet(id) scanname = info[0] scantarget = info[1] targetType = None if scanname == "" or scantarget == "" or len(scanconfig) == 0: return self.error("Something went wrong internally.") targetType = sf.targetType(scantarget) if targetType == None: # It must be a name, so wrap quotes around it scantarget = """ + scantarget + """ modlist = scanconfig['_modulesenabled'].split(',') templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup) return templ.render(pageid='NEWSCAN', types=types, docroot=self.docroot, modules=self.config['__modules__'], selectedmods=modlist, scanname=unicode(scanname, 'utf-8', errors='replace'), scantarget=unicode(scantarget, 'utf-8', errors='replace'))
def scandeletemulti(self, ids, confirm=None): dbh = SpiderFootDb(self.config) names = list() for id in ids.split(','): res = dbh.scanInstanceGet(id) names.append(res[0]) if res is None: return self.error("Scan ID not found (" + id + ").") if res[5] in ["RUNNING", "STARTING", "STARTED"]: return self.error("You cannot delete running scans.") if confirm is not None: for id in ids.split(','): dbh.scanInstanceDelete(id) raise cherrypy.HTTPRedirect("/") else: templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup) return templ.render(id=None, name=None, ids=ids.split(','), names=names, pageid="SCANLIST", docroot=self.docroot)
def resultsetfp(self, id, resultids, fp): dbh = SpiderFootDb(self.config) if fp not in ["0", "1"]: return json.dumps( ["ERROR", "No FP flag set or not set correctly."]) ids = json.loads(resultids) if not ids: return json.dumps(["ERROR", "No IDs supplied."]) # Cannot set FPs if a scan is not completed status = dbh.scanInstanceGet(id) if status[5] not in ["ABORTED", "FINISHED", "ERROR-FAILED"]: return json.dumps(["WARNING", "Scan must be in a finished state when " + \ "setting False Positives."]) # Make sure the user doesn't set something as non-FP when the # parent is set as an FP. if fp == "0": data = dbh.scanElementSourcesDirect(id, ids) for row in data: if str(row[14]) == "1": return json.dumps(["WARNING", "You cannot unset an element as False Positive " + \ "if a parent element is still False Positive."]) # Set all the children as FPs too.. it's only logical afterall, right? childs = dbh.scanElementChildrenAll(id, ids) allIds = ids + childs ret = dbh.scanResultsUpdateFP(id, allIds, fp) if not ret: return json.dumps(["ERROR", "Exception encountered."]) else: return json.dumps(["SUCCESS", ""])
def scanexportjsonmulti(self, ids): dbh = SpiderFootDb(self.config) scaninfo = dict() for id in ids.split(','): scan_name = dbh.scanInstanceGet(id)[0] if scan_name not in scaninfo: scaninfo[scan_name] = [] for row in dbh.scanResultEvent(id): lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0])) event_data = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "") source_data = str(row[2]) source_module = str(row[3]) event_type = row[4] false_positive = row[13] if event_type == "ROOT": continue scaninfo[scan_name].append({ "data": event_data, "type": event_type, "source_module": source_module, "source_data": source_data, "false_positive": false_positive, "lastseen": lastseen }) cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.json" cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8" cherrypy.response.headers['Pragma'] = "no-cache" return json.dumps(scaninfo)
def stopscanmulti(self, ids): global globalScanStatus # running scans dbh = SpiderFootDb(self.config) error = list() for id in ids.split(","): errState = False scaninfo = dbh.scanInstanceGet(id) if globalScanStatus.getStatus( id) == "FINISHED" or scaninfo[5] == "FINISHED": error.append("Scan '" + scaninfo[0] + "' is in a finished state. <a href='/scandelete?id=" + \ id + "&confirm=1'>Maybe you want to delete it instead?</a>") errState = True if not errState and (globalScanStatus.getStatus(id) == "ABORTED" or scaninfo[5] == "ABORTED"): error.append("Scan '" + scaninfo[0] + "' is already aborted.") errState = True if not errState and globalScanStatus.getStatus(id) is None: error.append("Scan '" + scaninfo[0] + "' is not actually running. A data consistency " + \ "error for this scan probably exists. <a href='/scandelete?id=" + \ id + "&confirm=1'>Click here to delete it.</a>") errState = True if not errState: globalScanStatus.setStatus(id, "ABORT-REQUESTED") templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup) return templ.render(pageid='SCANLIST', stoppedscan=True, errors=error, docroot=self.docroot)
def test_scanInstanceGet_should_return_scan_info(self): """ Test scanInstanceGet(self, instanceId) """ sfdb = SpiderFootDb(self.default_options, False) instance_id = "example instance id" scan_name = "example scan name" scan_target = "example scan target" sfdb.scanInstanceCreate(instance_id, scan_name, scan_target) scan_instance_get = sfdb.scanInstanceGet(instance_id) self.assertEqual(len(scan_instance_get), 6) self.assertIsInstance(scan_instance_get[0], str) self.assertEqual(scan_instance_get[0], scan_name) self.assertIsInstance(scan_instance_get[1], str) self.assertEqual(scan_instance_get[1], scan_target) self.assertIsInstance(scan_instance_get[2], float) self.assertIsInstance(scan_instance_get[3], float) self.assertIsInstance(scan_instance_get[4], float) self.assertIsInstance(scan_instance_get[5], str) self.assertEqual(scan_instance_get[5], 'CREATED')
def scandelete(self, id, confirm=None, raw=False): """Delete a scan Args: id (str): scan ID confirm: TBD raw: TBD """ dbh = SpiderFootDb(self.config) res = dbh.scanInstanceGet(id) if res is None: if raw: return json.dumps(["ERROR", "Scan ID not found."]) return self.error("Scan ID not found.") if confirm: dbh.scanInstanceDelete(id) if raw: return json.dumps(["SUCCESS", ""]) raise cherrypy.HTTPRedirect("/") templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup) return templ.render(id=id, name=str(res[0]), names=list(), ids=list(), pageid="SCANLIST", docroot=self.docroot)
def scanexportjsonmulti(self, ids): dbh = SpiderFootDb(self.config) scaninfo = dict() for id in ids.split(','): scan_name = dbh.scanInstanceGet(id)[0] if scan_name not in scaninfo: scaninfo[scan_name] = [] for row in dbh.scanResultEvent(id): lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0])) event_data = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "") source_data = str(row[2]) source_module = str(row[3]) event_type = row[4] false_positive = row[13] if event_type == "ROOT": continue scaninfo[scan_name].append({ "data": event_data, "type": event_type, "source_module": source_module, "source_data": source_data, "false_positive": false_positive, "lastseen": lastseen }) cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.json" cherrypy.response.headers['Content-Type'] = "application/json; charset=utf-8" cherrypy.response.headers['Pragma'] = "no-cache" return json.dumps(scaninfo)
def stopscan(self, id, cli=None): dbh = SpiderFootDb(self.config) scaninfo = dbh.scanInstanceGet(id) scanstatus = scaninfo[5] print(scanstatus) if scaninfo is None: if not cli: return self.error("Invalid scan ID.") else: return json.dumps(["ERROR", "Invalid scan ID."]) if scanstatus == "ABORTED": if not cli: return self.error("The scan is already aborted.") else: return json.dumps(["ERROR", "Scan already aborted."]) if not scanstatus == "RUNNING": if not cli: return self.error("The running scan is currently in the state '" + \ scanstatus + "', please try again later or restart " + \ " SpiderFoot.") else: return json.dumps( ["ERROR", "Scan in an invalid state for stopping."]) dbh.scanInstanceSet(id, status="ABORT-REQUESTED") if not cli: raise cherrypy.HTTPRedirect("/") else: return json.dumps(["SUCCESS", ""])
def scanopts(self, id): ret = dict() dbh = SpiderFootDb(self.config) ret['config'] = dbh.scanConfigGet(id) ret['configdesc'] = dict() for key in ret['config'].keys(): if ':' not in key: ret['configdesc'][key] = self.config['__globaloptdescs__'][key] else: [ modName, modOpt ] = key.split(':') if not modName in self.config['__modules__'].keys(): continue if not modOpt in self.config['__modules__'][modName]['optdescs'].keys(): continue ret['configdesc'][key] = self.config['__modules__'][modName]['optdescs'][modOpt] sf = SpiderFoot(self.config) meta = dbh.scanInstanceGet(id) if meta[3] != 0: started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(meta[3])) else: started = "Not yet" if meta[4] != 0: finished = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(meta[4])) else: finished = "Not yet" ret['meta'] = [meta[0], meta[1], meta[2], started, finished, meta[5]] return json.dumps(ret)
def stopscanmulti(self, ids): dbh = SpiderFootDb(self.config) error = list() for id in ids.split(","): errState = False scaninfo = dbh.scanInstanceGet(id) if scaninfo is None: return self.error("Invalid scan ID specified.") scanname = str(scaninfo[0]) scanstatus = scaninfo[5] if scanstatus == "FINISHED": error.append("Scan '" + scanname + "' is in a finished state. <a href='/scandelete?id=" + \ id + "&confirm=1'>Maybe you want to delete it instead?</a>") errState = True if not errState and scanstatus == "ABORTED": error.append("Scan '" + scanname + "' is already aborted.") errState = True if not errState: #set the scanstatus in the db to "ABORT-REQUESTED" dbh.scanInstanceSet(id, status="ABORT-REQUESTED") raise cherrypy.HTTPRedirect("/")
def stopscan(self, id): global globalScanStatus dbh = SpiderFootDb(self.config) scaninfo = dbh.scanInstanceGet(id) if scaninfo is None: return self.error("Invalid scan ID.") if globalScanStatus.getStatus(id) is None: return self.error("That scan is not actually running. A data consistency " + \ "error for this scan probably exists. <a href='/scandelete?id=" + \ id + "&confirm=1'>Click here to delete it.</a>") if globalScanStatus.getStatus(id) == "ABORTED": return self.error("The scan is already aborted.") if not globalScanStatus.getStatus(id) == "RUNNING": return self.error("The running scan is currently in the state '" + \ globalScanStatus.getStatus(id) + "', please try again later or restart " + \ " SpiderFoot.") globalScanStatus.setStatus(id, "ABORT-REQUESTED") templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup) return templ.render(pageid='SCANLIST', stoppedscan=True, docroot=self.docroot, errors=list())
def stopscanmulti(self, ids): #Removing All globalScanStatus Refs dbh = SpiderFootDb(self.config) error = list() for id in ids.split(","): errState = False scaninfo = dbh.scanInstanceGet(id) if scaninfo is None: return self.error("Invalid scan ID specified.") scanname = str(scaninfo[0]) scanstatus = scaninfo[5] if scanstatus == "FINISHED": error.append("Scan '" + scanname + "' is in a finished state. <a href='/scandelete?id=" + \ id + "&confirm=1'>Maybe you want to delete it instead?</a>") errState = True if not errState and scanstatus == "ABORTED": error.append("Scan '" + scanname + "' is already aborted.") errState = True if not errState: #set the scanstatus in the db to "ABORT-REQUESTED" dbh.scanInstanceSet(id, status="ABORT-REQUESTED") templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup) return templ.render(pageid='SCANLIST', stoppedscan=True, errors=error, docroot=self.docroot)
def scandelete(self, id, confirm=None, raw=False): dbh = SpiderFootDb(self.config) res = dbh.scanInstanceGet(id) if res is None: if not raw: return self.error("Scan ID not found.") else: return json.dumps(["ERROR", "Scan ID not found."]) if confirm is not None: dbh.scanInstanceDelete(id) if not raw: raise cherrypy.HTTPRedirect("/") else: return json.dumps(["SUCCESS", ""]) else: templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup) return templ.render(id=id, name=unicode(res[0], 'utf-8', errors='replace'), names=list(), ids=list(), pageid="SCANLIST", docroot=self.docroot)
def scanopts(self, id): ret = dict() dbh = SpiderFootDb(self.config) ret['config'] = dbh.scanConfigGet(id) ret['configdesc'] = dict() for key in ret['config'].keys(): if ':' not in key: ret['configdesc'][key] = self.config['__globaloptdescs__'][key] else: [modName, modOpt] = key.split(':') if modName not in self.config['__modules__'].keys(): continue if modOpt not in self.config['__modules__'][modName]['optdescs'].keys(): continue ret['configdesc'][key] = self.config['__modules__'][modName]['optdescs'][modOpt] sf = SpiderFoot(self.config) meta = dbh.scanInstanceGet(id) if meta[3] != 0: started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(meta[3])) else: started = "Not yet" if meta[4] != 0: finished = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(meta[4])) else: finished = "Not yet" ret['meta'] = [meta[0], meta[1], meta[2], started, finished, meta[5]] return json.dumps(ret)
def stopscan(self, id, cli=None): dbh = SpiderFootDb(self.config) scaninfo = dbh.scanInstanceGet(id) scanstatus = scaninfo[5] print(scanstatus) if scaninfo is None: if not cli: return self.error("Invalid scan ID.") else: return json.dumps(["ERROR", "Invalid scan ID."]) if scanstatus == "ABORTED": if not cli: return self.error("The scan is already aborted.") else: return json.dumps(["ERROR", "Scan already aborted."]) if not scanstatus == "RUNNING": if not cli: return self.error("The running scan is currently in the state '" + \ scanstatus + "', please try again later or restart " + \ " SpiderFoot.") else: return json.dumps( ["ERROR", "Scan in an invalid state for stopping."]) dbh.scanInstanceSet(id, status="ABORT-REQUESTED") if not cli: templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup) return templ.render(pageid='SCANLIST', stoppedscan=True, docroot=self.docroot, errors=list()) else: return json.dumps(["SUCCESS", ""])
def resultsetfp(self, id, resultids, fp): dbh = SpiderFootDb(self.config) if fp not in ["0", "1"]: return json.dumps(["ERROR", "No FP flag set or not set correctly."]) ids = json.loads(resultids) if not ids: return json.dumps(["ERROR", "No IDs supplied."]) # Cannot set FPs if a scan is not completed status = dbh.scanInstanceGet(id) if status[5] not in [ "ABORTED", "FINISHED", "ERROR-FAILED" ]: return json.dumps(["WARNING", "Scan must be in a finished state when " + \ "setting False Positives."]) # Make sure the user doesn't set something as non-FP when the # parent is set as an FP. if fp == "0": data = dbh.scanElementSourcesDirect(id, ids) for row in data: if str(row[14]) == "1": return json.dumps(["WARNING", "You cannot unset an element as False Positive " + \ "if a parent element is still False Positive."]); # Set all the children as FPs too.. it's only logical afterall, right? childs = dbh.scanElementChildrenAll(id, ids) allIds = ids + childs ret = dbh.scanResultsUpdateFP(id, allIds, fp) if not ret: return json.dumps(["ERROR", "Exception encountered."]) else: return json.dumps(["SUCCESS", ""])
def stopscanmulti(self, ids): global globalScanStatus # running scans dbh = SpiderFootDb(self.config) error = list() for id in ids.split(","): errState = False scaninfo = dbh.scanInstanceGet(id) if globalScanStatus.getStatus(id) == "FINISHED" or scaninfo[5] == "FINISHED": error.append("Scan '" + scaninfo[0] + "' is in a finished state. <a href='/scandelete?id=" + \ id + "&confirm=1'>Maybe you want to delete it instead?</a>") errState = True if not errState and (globalScanStatus.getStatus(id) == "ABORTED" or scaninfo[5] == "ABORTED"): error.append("Scan '" + scaninfo[0] + "' is already aborted.") errState = True if not errState and globalScanStatus.getStatus(id) is None: error.append("Scan '" + scaninfo[0] + "' is not actually running. A data consistency " + \ "error for this scan probably exists. <a href='/scandelete?id=" + \ id + "&confirm=1'>Click here to delete it.</a>") errState = True if not errState: globalScanStatus.setStatus(id, "ABORT-REQUESTED") templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup) return templ.render(pageid='SCANLIST', stoppedscan=True, errors=error, docroot=self.docroot)
def clonescan(self, id): sf = SpiderFoot(self.config) dbh = SpiderFootDb(self.config) types = dbh.eventTypes() info = dbh.scanInstanceGet(id) scanconfig = dbh.scanConfigGet(id) scanname = info[0] scantarget = info[1] targetType = None if scanname == "" or scantarget == "" or len(scanconfig) == 0: return self.error("Something went wrong internally.") targetType = sf.targetType(scantarget) if targetType == None: # It must be a name, so wrap quotes around it scantarget = """ + scantarget + """ modlist = scanconfig['_modulesenabled'].split(',') templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup) return templ.render(pageid='NEWSCAN', types=types, docroot=self.docroot, modules=self.config['__modules__'], selectedmods=modlist, scanname=str(scanname), scantarget=str(scantarget))
def scaninfo(self, id): dbh = SpiderFootDb(self.config) res = dbh.scanInstanceGet(id) if res == None: return self.error("Scan ID not found.") templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup) return templ.render(id=id, name=res[0], status=res[5], pageid='SCANLIST')
def scaninfo(self, id): dbh = SpiderFootDb(self.config) res = dbh.scanInstanceGet(id) if res is None: return self.error("Scan ID not found.") templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup, disable_unicode=True, input_encoding='utf-8') return templ.render(id=id, name=cgi.escape(res[0]), status=res[5], docroot=self.docroot, pageid="SCANLIST")
def scaninfo(self, id): dbh = SpiderFootDb(self.config) res = dbh.scanInstanceGet(id) if res is None: return self.error("Scan ID not found.") templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup, disable_unicode=True, input_encoding='utf-8') return templ.render(id=id, name=cgi.escape(res[0]), status=res[5], docroot=self.docroot, pageid="SCANLIST")
def scanstatus(self, id): dbh = SpiderFootDb(self.config) data = dbh.scanInstanceGet(id) created = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[2])) started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[3])) ended = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[4])) retdata = [data[0], data[1], created, started, ended, data[5]] return json.dumps(retdata)
def scanstatus(self, id): dbh = SpiderFootDb(self.config) data = dbh.scanInstanceGet(id) created = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[2])) started = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[3])) ended = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(data[4])) retdata = [data[0], data[1], created, started, ended, data[5]] return json.dumps(retdata)
def rerunscanmulti(self, ids): # Snapshot the current configuration to be used by the scan cfg = deepcopy(self.config) modopts = dict() # Not used yet as module options are set globally modlist = list() sf = SpiderFoot(cfg) dbh = SpiderFootDb(cfg) for id in ids.split(","): info = dbh.scanInstanceGet(id) scanconfig = dbh.scanConfigGet(id) scanname = info[0] scantarget = info[1] targetType = None if len(scanconfig) == 0: return self.error("Something went wrong internally.") modlist = scanconfig['_modulesenabled'].split(',') if "sfp__stor_stdout" in modlist: modlist.remove("sfp__stor_stdout") targetType = sf.targetType(scantarget) if targetType == None: # Should never be triggered for a re-run scan.. return self.error("Invalid target type. Could not recognize it as " + \ "a human name, IP address, IP subnet, ASN, domain name or host name.") # Start running a new scan newId = sf.genScanInstanceGUID(scanname) p = mp.Process(target=SpiderFootScanner, args=(scanname, scantarget.lower(), targetType, newId, modlist, cfg, modopts)) p.start() # Wait until the scan has initialized while dbh.scanInstanceGet(newId) == None: print("[info] Waiting for the scan to initialize...") time.sleep(1) templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup) return templ.render(rerunscans=True, docroot=self.docroot, pageid="SCANLIST")
def test_scanInstanceGet_argument_instanceId_of_invalid_type_should_raise_TypeError(self): """ Test scanInstanceGet(self, instanceId) """ sfdb = SpiderFootDb(self.default_options, False) invalid_types = [None, list(), dict(), int()] for invalid_type in invalid_types: with self.subTest(invalid_type=invalid_type): with self.assertRaises(TypeError) as cm: scan_instance_get = sfdb.scanInstanceGet(invalid_type)
def test_scan_instance_get_should_return_a_list(self): """ Test scanInstanceGet(self, instanceId) """ sfdb = SpiderFootDb(self.default_options, False) scan_instance = 'example scan instance' sfdb.scanInstanceSet(scan_instance, None, None, None) scan_instance_get = sfdb.scanInstanceGet(scan_instance) self.assertIsInstance(scan_instance_get, list)
def scandelete(self, id, confirm=None): dbh = SpiderFootDb(self.config) res = dbh.scanInstanceGet(id) if res == None: return self.error("Scan ID not found.") if confirm != None: dbh.scanInstanceDelete(id) raise cherrypy.HTTPRedirect("/") else: templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup) return templ.render(id=id, name=res[0])
def scanviz(self, id, gexf="0"): dbh = SpiderFootDb(self.config) sf = SpiderFoot(self.config) data = dbh.scanResultEvent(id, filterFp=True) scan = dbh.scanInstanceGet(id) root = scan[1] if gexf != "0": cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.gexf" cherrypy.response.headers['Content-Type'] = "application/gexf" cherrypy.response.headers['Pragma'] = "no-cache" return sf.buildGraphGexf([root], "SpiderFoot Export", data) else: return sf.buildGraphJson([root], data)
def scandelete(self, id, confirm=None): dbh = SpiderFootDb(self.config) res = dbh.scanInstanceGet(id) if res is None: return self.error("Scan ID not found.") if confirm is not None: dbh.scanInstanceDelete(id) raise cherrypy.HTTPRedirect("/") else: templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup) return templ.render(id=id, name=res[0], names=list(), ids=list(), pageid="SCANLIST", docroot=self.docroot)
def scanviz(self, id, gexf="0"): types = list() dbh = SpiderFootDb(self.config) sf = SpiderFoot(self.config) data = dbh.scanResultEvent(id, filterFp=True) scan = dbh.scanInstanceGet(id) root = scan[1] if gexf != "0": cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.gexf" cherrypy.response.headers['Content-Type'] = "application/gexf" cherrypy.response.headers['Pragma'] = "no-cache" return sf.buildGraphGexf([root], "SpiderFoot Export", data) else: return sf.buildGraphJson([root], data)
def scanexportjsonmulti(self, ids): dbh = SpiderFootDb(self.config) scaninfo = list() scan_name = "" for id in ids.split(','): scan = dbh.scanInstanceGet(id) if scan is None: continue scan_name = scan[0] for row in dbh.scanResultEvent(id): lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0])) event_data = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "") source_data = str(row[2]) source_module = str(row[3]) event_type = row[4] false_positive = row[13] if event_type == "ROOT": continue scaninfo.append({ "data": event_data, "event_type": event_type, "module": source_module, "source_data": source_data, "false_positive": false_positive, "last_seen": lastseen, "scan_name": scan_name, "scan_target": scan[1] }) if len(ids.split(',')) > 1 or scan_name == "": fname = "SpiderFoot.json" else: fname = scan_name + "-SpiderFoot.json" cherrypy.response.headers[ 'Content-Disposition'] = "attachment; filename=" + fname cherrypy.response.headers[ 'Content-Type'] = "application/json; charset=utf-8" cherrypy.response.headers['Pragma'] = "no-cache" return json.dumps(scaninfo).encode("utf-8")
def scanvizmulti(self, ids, gexf="1"): dbh = SpiderFootDb(self.config) sf = SpiderFoot(self.config) data = list() roots = list() for id in ids.split(','): data = data + dbh.scanResultEvent(id, filterFp=True) roots.append(dbh.scanInstanceGet(id)[1]) if gexf != "0": cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.gexf" cherrypy.response.headers['Content-Type'] = "application/gexf" cherrypy.response.headers['Pragma'] = "no-cache" return sf.buildGraphGexf(roots, "SpiderFoot Export", data) else: # Not implemented yet return None
def clonescan(self, id): dbh = SpiderFootDb(self.config) types = dbh.eventTypes() info = dbh.scanInstanceGet(id) scanconfig = dbh.scanConfigGet(id) scanname = info[0] scantarget = info[1] targetType = None if scanname == "" or scantarget == "" or len(scanconfig) == 0: return self.error("Something went wrong internally.") modlist = scanconfig['_modulesenabled'].split(',') templ = Template(filename='dyn/newscan.tmpl', lookup=self.lookup) return templ.render(pageid='NEWSCAN', types=types, docroot=self.docroot, modules=self.config['__modules__'], selectedmods=modlist, scanname=scanname, scantarget=scantarget)
def scanvizmulti(self, ids, gexf="1"): types = list() dbh = SpiderFootDb(self.config) sf = SpiderFoot(self.config) data = list() roots = list() for id in ids.split(','): data = data + dbh.scanResultEvent(id, filterFp=True) roots.append(dbh.scanInstanceGet(id)[1]) if gexf != "0": cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.gexf" cherrypy.response.headers['Content-Type'] = "application/gexf" cherrypy.response.headers['Pragma'] = "no-cache" return sf.buildGraphGexf(roots, "SpiderFoot Export", data) else: # Not implemented yet return None
def scandelete(self, id, confirm=None, raw=False): dbh = SpiderFootDb(self.config) res = dbh.scanInstanceGet(id) if res is None: if not raw: return self.error("Scan ID not found.") else: return json.dumps(["ERROR", "Scan ID not found."]) if confirm is not None: dbh.scanInstanceDelete(id) if not raw: raise cherrypy.HTTPRedirect("/") else: return json.dumps(["SUCCESS", ""]) else: templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup) return templ.render(id=id, name=unicode(res[0], 'utf-8', errors='replace'), names=list(), ids=list(), pageid="SCANLIST", docroot=self.docroot)
def rerunscan(self, id): # Snapshot the current configuration to be used by the scan cfg = deepcopy(self.config) modopts = dict() # Not used yet as module options are set globally modlist = list() sf = SpiderFoot(cfg) dbh = SpiderFootDb(cfg) info = dbh.scanInstanceGet(id) scanconfig = dbh.scanConfigGet(id) scanname = info[0] scantarget = info[1] targetType = None if len(scanconfig) == 0: return self.error("Something went wrong internally.") modlist = scanconfig['_modulesenabled'].split(',') targetType = sf.targetType(scantarget) if targetType == None: # It must then be a name, as a re-run scan should always have a clean # target. targetType = "HUMAN_NAME" if targetType != "HUMAN_NAME": scantarget = scantarget.lower() # Start running a new scan newId = sf.genScanInstanceGUID(scanname) t = SpiderFootScanner(scanname, scantarget, targetType, newId, modlist, cfg, modopts) t.start() # Wait until the scan has initialized while globalScanStatus.getStatus(newId) == None: print "[info] Waiting for the scan to initialize..." time.sleep(1) templ = Template(filename='dyn/scaninfo.tmpl', lookup=self.lookup) return templ.render(id=newId, name=unicode(scanname, 'utf-8', errors='replace'), docroot=self.docroot, status=globalScanStatus.getStatus(newId), pageid="SCANLIST")
def scaneventresultexportmulti(self, ids, dialect="excel"): dbh = SpiderFootDb(self.config) scaninfo = dict() data = list() for id in ids.split(','): scaninfo[id] = dbh.scanInstanceGet(id) data = data + dbh.scanResultEvent(id) fileobj = StringIO() parser = csv.writer(fileobj, dialect=dialect) parser.writerow(["Scan Name", "Updated", "Type", "Module", "Source", "Data"]) for row in data: if row[4] == "ROOT": continue lastseen = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(row[0])) datafield = str(row[1]).replace("<SFURL>", "").replace("</SFURL>", "") parser.writerow([scaninfo[row[12]][0], lastseen, str(row[4]), str(row[3]), str(row[2]), datafield]) cherrypy.response.headers['Content-Disposition'] = "attachment; filename=SpiderFoot.csv" cherrypy.response.headers['Content-Type'] = "application/csv" cherrypy.response.headers['Pragma'] = "no-cache" return fileobj.getvalue()
def scandeletemulti(self, ids, confirm=None): dbh = SpiderFootDb(self.config) names = list() for id in ids.split(','): res = dbh.scanInstanceGet(id) names.append(res[0]) if res is None: return self.error("Scan ID not found (" + id + ").") if res[5] in [ "RUNNING", "STARTING", "STARTED" ]: return self.error("You cannot delete running scans.") if confirm is not None: for id in ids.split(','): dbh.scanInstanceDelete(id) raise cherrypy.HTTPRedirect("/") else: templ = Template(filename='dyn/scandelete.tmpl', lookup=self.lookup) return templ.render(id=None, name=None, ids=ids.split(','), names=names, pageid="SCANLIST", docroot=self.docroot)
def stopscan(self, id, cli=None): global globalScanStatus dbh = SpiderFootDb(self.config) scaninfo = dbh.scanInstanceGet(id) if scaninfo is None: if not cli: return self.error("Invalid scan ID.") else: return json.dumps(["ERROR", "Invalid scan ID."]) if globalScanStatus.getStatus(id) is None: if not cli: return self.error("That scan is not actually running. A data consistency " + \ "error for this scan probably exists. <a href='/scandelete?id=" + \ id + "&confirm=1'>Click here to delete it.</a>") else: return json.dumps(["ERROR", "Scan doesn't appear to be running."]) if globalScanStatus.getStatus(id) == "ABORTED": if not cli: return self.error("The scan is already aborted.") else: return json.dumps(["ERROR", "Scan already aborted."]) if not globalScanStatus.getStatus(id) == "RUNNING": if not cli: return self.error("The running scan is currently in the state '" + \ globalScanStatus.getStatus(id) + "', please try again later or restart " + \ " SpiderFoot.") else: return json.dumps(["ERROR", "Scan in an invalid state for stopping."]) globalScanStatus.setStatus(id, "ABORT-REQUESTED") if not cli: templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup) return templ.render(pageid='SCANLIST', stoppedscan=True, docroot=self.docroot, errors=list()) else: return json.dumps(["SUCCESS", ""])
def rerunscanmulti(self, ids): # Snapshot the current configuration to be used by the scan cfg = deepcopy(self.config) modopts = dict() # Not used yet as module options are set globally modlist = list() sf = SpiderFoot(cfg) dbh = SpiderFootDb(cfg) for id in ids.split(","): info = dbh.scanInstanceGet(id) scanconfig = dbh.scanConfigGet(id) scanname = info[0] scantarget = info[1] targetType = None if len(scanconfig) == 0: return self.error("Something went wrong internally.") modlist = scanconfig['_modulesenabled'].split(',') targetType = sf.targetType(scantarget) if targetType == None: # Should never be triggered for a re-run scan.. return self.error("Invalid target type. Could not recognize it as " + \ "a human name, IP address, IP subnet, ASN, domain name or host name.") # Start running a new scan newId = sf.genScanInstanceGUID(scanname) t = SpiderFootScanner(unicode(scanname, 'utf-8', errors='replace'), unicode(scantarget, 'utf-8', errors='replace').lower(), targetType, newId, modlist, cfg, modopts) t.start() # Wait until the scan has initialized while globalScanStatus.getStatus(newId) == None: print "[info] Waiting for the scan to initialize..." time.sleep(1) templ = Template(filename='dyn/scanlist.tmpl', lookup=self.lookup) return templ.render(rerunscans=True, docroot=self.docroot, pageid="SCANLIST")
if args.debug: cfg['_debug'] = True else: cfg['_debug'] = False # If strict mode is enabled, filter the output from modules. if args.x and args.t: cfg['__outputfilter'] = args.t.split(",") t = SpiderFootScanner(target, target, targetType, scanId, modlist, cfg, dict()) t.daemon = True t.start() while True: info = dbh.scanInstanceGet(scanId) if not info: time.sleep(1) continue if info[5] in [ "ERROR-FAILED", "ABORT-REQUESTED", "ABORTED", "FINISHED" ]: if not args.q: print "[*] Scan completed with status " + info[5] sys.exit(0) time.sleep(1) sys.exit(0) # Start the web server so you can start looking at results print "Starting web server at http://" + sfConfig['__webaddr'] + \ ":" + str(sfConfig['__webport']) + sfConfig['__docroot'] + " ..."