def setupErrorLogs(self): fileName = "errors-{0}.log".format(time.strftime("%y-%m-%d_%H-%M-%S")) self.errorLogPath = FileUtils.build_path( FileUtils.build_path(self.savePath, "logs", fileName)) try: self.errorLog = open(self.errorLogPath, "w") except PermissionError: self.output.error( "Couldn't create the error log. Try running again with highest permission" ) sys.exit(1)
def getBlacklists(self): blacklists = {} for status in [400, 403, 500]: blacklistFileName = FileUtils.buildPath(self.script_path, 'db') blacklistFileName = FileUtils.buildPath(blacklistFileName, '{}_blacklist.txt'.format(status)) if not FileUtils.canRead(blacklistFileName): # Skip if cannot read file continue blacklists[status] = [] for line in FileUtils.getLines(blacklistFileName): # Skip comments if line.lstrip().startswith('#'): continue blacklists[status].append(line) return blacklists
def validate_path(self, path): if not FileUtils.exists(path): self.output.error("{} does not exist".format(path)) exit(1) if FileUtils.exists(path) and not FileUtils.is_dir(path): self.output.error( "{} is a file, should be a directory".format(path)) exit(1) if not FileUtils.can_write(path): self.output.error("Directory {} is not writable".format(path)) exit(1) return True
def setup_batch_reports(self): self.batch = True if not self.arguments.output_file: self.batch_session = "BATCH-{0}".format( time.strftime("%y-%m-%d_%H-%M-%S")) self.batch_directory_path = FileUtils.build_path( self.save_path, self.batch_session) if not FileUtils.exists(self.batch_directory_path): FileUtils.create_directory(self.batch_directory_path) if not FileUtils.exists(self.batch_directory_path): self.output.error( "Couldn't create batch folder at {}".format( self.batch_directory_path)) sys.exit(1)
def matchCallback(self, path): self.index += 1 if path.status: if path.status not in self.excludeStatusCodes and ( not self.includeStatusCodes or path.status in self.includeStatusCodes ) and (not self.blacklists.get(path.status) or path.path not in self.blacklists.get(path.status)) and ( not self.excludeSizes or FileUtils.size_human( len(path.response.body)).strip() not in self.excludeSizes ) and (not self.minimumResponseSize or self.minimumResponseSize < len(path.response.body) ) and (not self.maximumResponseSize or self.maximumResponseSize > len( path.response.body)): for excludeText in self.excludeTexts: if excludeText in path.response.body.decode('iso8859-1'): del path return for excludeRegexp in self.excludeRegexps: if (re.search(excludeRegexp, path.response.body.decode('iso8859-1')) is not None): del path return pathIsInScanSubdirs = False addedToQueue = False if self.scanSubdirs: for subdir in self.scanSubdirs: if subdir == path.path + "/": pathIsInScanSubdirs = True if not self.recursive and not pathIsInScanSubdirs and "?" not in path.path: if path.response.redirect: addedToQueue = self.addRedirectDirectory(path) else: addedToQueue = self.addDirectory(path.path) self.output.statusReport(path.path, path.response, self.arguments.full_url, addedToQueue) if self.arguments.matches_proxy: self.requester.request(path.path, proxy=self.arguments.matches_proxy) newPath = "{}{}".format(self.currentDirectory, path.path) self.reportManager.addPath(newPath, path.status, path.response) self.reportManager.save() del path
def getBlacklists(self): reext = re.compile('\%ext\%', re.IGNORECASE) reextdot = re.compile('\.\%ext\%', re.IGNORECASE) blacklists = {} for status in [400, 403, 500]: blacklistFileName = FileUtils.buildPath(self.script_path, "db") blacklistFileName = FileUtils.buildPath( blacklistFileName, "{}_blacklist.txt".format(status)) if not FileUtils.canRead(blacklistFileName): # Skip if cannot read file continue blacklists[status] = [] for line in FileUtils.getLines(blacklistFileName): # Skip comments if line.lstrip().startswith("#"): continue # The same with Dictionary.py if line.startswith("/"): line = line[1:] # Classic dirsearch blacklist processing (with %EXT% keyword) if "%ext%" in line.lower(): for extension in self.arguments.extensions: if self.arguments.noDotExtensions: entry = reextdot.sub(extension, line) else: entry = line entry = reext.sub(extension, entry) blacklists[status].append(entry) # Forced extensions is not used here because -r is only used for wordlist (in documentation), # applying in blacklist may create false negatives else: blacklists[status].append(line) return blacklists
def getSavePath(self): basePath = None dirPath = None basePath = os.path.expanduser('~') if os.name == 'nt': dirPath = "dirsearch" else: dirPath = ".dirsearch" return FileUtils.buildPath(basePath, dirPath)
def getSavePath(self): basePath = None dirPath = None basePath = os.path.expanduser("~") if os.name == "nt": dirPath = "dirsearch" else: dirPath = ".dirsearch" return FileUtils.buildPath(basePath, dirPath)
def setupReports(self, requester): if self.arguments.autoSave: basePath = ('/' if requester.basePath is '' else requester.basePath) basePath = basePath.replace(os.path.sep, '.')[1:-1] fileName = None directoryPath = None if self.batch: fileName = requester.host directoryPath = self.batchDirectoryPath else: fileName = ('{}_'.format(basePath) if basePath is not '' else '') fileName += time.strftime('%y-%m-%d_%H-%M-%S') directoryPath = FileUtils.buildPath(self.savePath,'reports', requester.host) outputFile = FileUtils.buildPath(directoryPath, fileName) if FileUtils.exists(outputFile): i = 2 while FileUtils.exists(outputFile + "_" + str(i)): i += 1 outputFile += "_" + str(i) if not FileUtils.exists(directoryPath): FileUtils.createDirectory(directoryPath) if not FileUtils.exists(directoryPath): self.output.error("Couldn't create reports folder {}".format(directoryPath)) sys.exit(1) if FileUtils.canWrite(directoryPath): report = None if self.arguments.autoSaveFormat == 'simple': report = SimpleReport(requester.host, requester.port, requester.protocol, requester.basePath, outputFile) if self.arguments.autoSaveFormat == 'json': report = JSONReport(requester.host, requester.port, requester.protocol, requester.basePath, outputFile) else: report = PlainTextReport(requester.host, requester.port, requester.protocol, requester.basePath, outputFile) self.reportManager.addOutput(report) else: self.output.error("Can't write reports to {}".format(directoryPath)) sys.exit(1) if self.arguments.simpleOutputFile is not None: self.reportManager.addOutput(SimpleReport(requester.host, requester.port, requester.protocol, requester.basePath, self.arguments.simpleOutputFile)) if self.arguments.plainTextOutputFile is not None: self.reportManager.addOutput(PlainTextReport(requester.host, requester.port, requester.protocol, requester.basePath, self.arguments.plainTextOutputFile)) if self.arguments.jsonOutputFile is not None: self.reportManager.addOutput(JSONReport(requester.host, requester.port, requester.protocol, requester.basePath, self.arguments.jsonOutputFile))
def setupReports(self, requester): if self.arguments.autoSave: basePath = ('/' if requester.basePath is '' else requester.basePath) basePath = basePath.replace(os.path.sep, '.')[1:-1] fileName = None directoryPath = None if self.batch: fileName = requester.host directoryPath = self.batchDirectoryPath else: fileName = ('{}_'.format(basePath) if basePath is not '' else '') fileName += time.strftime('%y-%m-%d_%H-%M-%S') directoryPath = FileUtils.buildPath(self.script_path, 'reports', requester.host) outputFile = FileUtils.buildPath(directoryPath, fileName) if FileUtils.exists(outputFile): i = 2 while FileUtils.exists(outputFile + "_" + str(i)): i += 1 outputFile += "_" + str(i) if not FileUtils.exists(directoryPath): FileUtils.createDirectory(directoryPath) if not FileUtils.exists(directoryPath): self.output.error("Couldn't create reports folder {}".format(directoryPath)) sys.exit(1) if FileUtils.canWrite(directoryPath): report = None if self.arguments.autoSaveFormat == 'simple': report = SimpleReport(requester.host, requester.port, requester.protocol, requester.basePath, outputFile) if self.arguments.autoSaveFormat == 'json': report = JSONReport(requester.host, requester.port, requester.protocol, requester.basePath, outputFile) else: report = PlainTextReport(requester.host, requester.port, requester.protocol, requester.basePath, outputFile) self.reportManager.addOutput(report) else: self.output.error("Can't write reports to {}".format(directoryPath)) sys.exit(1) if self.arguments.simpleOutputFile is not None: self.reportManager.addOutput(SimpleReport(requester.host, requester.port, requester.protocol, requester.basePath, self.arguments.simpleOutputFile)) if self.arguments.plainTextOutputFile is not None: self.reportManager.addOutput(PlainTextReport(requester.host, requester.port, requester.protocol, requester.basePath, self.arguments.plainTextOutputFile)) if self.arguments.jsonOutputFile is not None: self.reportManager.addOutput(JSONReport(requester.host, requester.port, requester.protocol, requester.basePath, self.arguments.jsonOutputFile))
def setup_reports(self): if self.arguments.output_file: output_file = FileUtils.get_abs_path(self.arguments.output_file) self.output.output_file(output_file) else: if self.batch: file_name = "BATCH" file_name += self.get_output_extension() directory_path = self.batch_directory_path else: local_requester = Requester(self.url_list[0]) file_name = ("{}_".format( local_requester.base_path.replace(os.path.sep, ".")[:-1])) file_name += time.strftime("%y-%m-%d_%H-%M-%S") file_name += self.get_output_extension() directory_path = FileUtils.build_path(self.save_path, local_requester.host) output_file = FileUtils.build_path(directory_path, file_name) if FileUtils.exists(output_file): i = 2 while FileUtils.exists(output_file + "_" + str(i)): i += 1 output_file += "_" + str(i) if not FileUtils.exists(directory_path): FileUtils.create_directory(directory_path) if not FileUtils.exists(directory_path): self.output.error( "Couldn't create the reports folder at {}".format( directory_path)) sys.exit(1) self.output.output_file(output_file) if self.arguments.output_file and self.arguments.output_format: self.report_manager = ReportManager(self.arguments.output_format, self.arguments.output_file) elif self.arguments.output_format: self.report_manager = ReportManager(self.arguments.output_format, output_file) else: self.report_manager = ReportManager("plain", output_file)
def setupReports(self): if self.arguments.outputFile is not None: outputFile = FileUtils.get_abs_path(self.arguments.outputFile) self.output.outputFile(outputFile) else: if self.batch: fileName = "BATCH" fileName += self.getOutputExtension() directoryPath = self.batchDirectoryPath else: localRequester = Requester(self.urlList[0]) fileName = ('{}_'.format( localRequester.basePath.replace(os.path.sep, ".")[:-1])) fileName += time.strftime('%y-%m-%d_%H-%M-%S') fileName += self.getOutputExtension() directoryPath = FileUtils.build_path(self.savePath, 'reports', localRequester.host) outputFile = FileUtils.build_path(directoryPath, fileName) if FileUtils.exists(outputFile): i = 2 while FileUtils.exists(outputFile + "_" + str(i)): i += 1 outputFile += "_" + str(i) if not FileUtils.exists(directoryPath): FileUtils.create_directory(directoryPath) if not FileUtils.exists(directoryPath): self.output.error( "Couldn't create the reports folder at {}".format( directoryPath)) sys.exit(1) self.output.outputFile(outputFile) if self.arguments.outputFile and self.arguments.outputFormat: self.reportManager = ReportManager(self.arguments.outputFormat, self.arguments.outputFile) elif self.arguments.outputFormat: self.reportManager = ReportManager(self.arguments.outputFormat, outputFile) else: self.reportManager = ReportManager("plain", outputFile)
def __init__(self, script_path, arguments, output): global VERSION program_banner = open( FileUtils.buildPath(script_path, "lib", "controller", "banner.txt")).read().format(**VERSION) self.script_path = script_path self.exit = False self.arguments = arguments self.output = output self.savePath = self.script_path self.doneDirs = []
def process(args): output = Output() logging.basicConfig(format='[%(levelname)s]: %(message)s', level=logging.INFO) logger = logging.getLogger("bakspider") if args.debug: logger.info('Debug mode is enabled, output will be verbose.') else: logger.disabled = True if not WebUtils.is_valid_target_url(args.url): output.error( "The URL you specified is not in the correct format, see examples:" ) print("\nValid examples:") output.status("http://www.example.com/") output.status("http://example.com/") print("\nInvalid examples:") output.negative("www.example.com") output.negative("http://www.example.com") sys.exit(1) # Check host is online if WebUtils.is_200_response(args.url): output.page_found("{0} -> Beginning scan...".format(args.url), False) else: output.error( "The URL you specified is returning an invalid response code.") sys.exit(1) website = SiteScanner(args.url, output, args.threads) if args.dir: dir_scan = DirScanner(args.url, args.dir, output) website.additional_dirs = dir_scan.scan(args.threads) website.backup_extensions = FileUtils.read_file_into_array(args.bakext) website.whitelist_extensions = FileUtils.read_file_into_array(args.ext) website.begin_scan()
def setupErrorLogs(self): fileName = "errors-{0}.log".format(time.strftime('%y-%m-%d_%H-%M-%S')) self.errorLogPath = FileUtils.buildPath( FileUtils.buildPath(self.savePath, "logs", fileName)) logs = FileUtils.buildPath(self.savePath, "logs") if not FileUtils.exists(logs): FileUtils.createDirectory(logs) self.errorLogPath = FileUtils.buildPath(logs, fileName) self.errorLog = open(self.errorLogPath, "w")
def valid(self, path): if not path: return False if path.status in self.exclude_status_codes: return False if self.include_status_codes and path.status not in self.include_status_codes: return False if self.blacklists.get(path.status) and path.path in self.blacklists.get(path.status): return False if self.exclude_sizes and FileUtils.size_human(len(path.response.body)).strip() in self.exclude_sizes: return False if self.minimum_response_size and self.minimum_response_size > len(path.response.body): return False if self.maximum_response_size and self.maximum_response_size < len(path.response.body): return False for exclude_text in self.exclude_texts: if exclude_text in path.response.body.decode('iso8859-1'): return False for exclude_regexp in self.exclude_regexps: if ( re.search(exclude_regexp, path.response.body.decode('iso8859-1')) is not None ): return False for exclude_redirect in self.exclude_redirects: if path.response.redirect and ( ( re.match(exclude_redirect, path.response.redirect) is not None ) or ( exclude_redirect in path.response.redirect ) ): return False return True
def generate(self): template_file = os.path.dirname(os.path.realpath( __file__)) + '/templates/html_report_template.html' mytemplate = Template(filename=template_file) metadata = {"command": " ".join(sys.argv), "date": time.ctime()} results = [] for entry in self.entries: for e in entry.results: headerName = "{0}://{1}:{2}/{3}".format( entry.protocol, entry.host, entry.port, entry.basePath) statusColorClass = '' if e.status >= 200 and e.status <= 299: statusColorClass = 'text-success' elif e.status >= 300 and e.status <= 399: statusColorClass = 'text-warning' elif e.status >= 400 and e.status <= 599: statusColorClass = 'text-danger' results.append({ "url": headerName + e.path, "path": e.path, "status": e.status, "statusColorClass": statusColorClass, "contentLength": FileUtils.size_human(e.getContentLength()), "contentType": e.response.headers.get("content-type"), "redirect": e.response.redirect }) return mytemplate.render(metadata=metadata, results=json.dumps(results))
def __init__(self, script_path, arguments, output): global VERSION program_banner = (open( FileUtils.buildPath(script_path, "lib", "controller", "banner.txt")).read().format(**VERSION)) self.script_path = script_path self.exit = False self.arguments = arguments self.output = output self.savePath = self.script_path self.doneDirs = [] self.recursive_level_max = self.arguments.recursive_level_max if self.arguments.httpmethod.lower() not in [ "get", "head", "post", "put", "patch", "options", "delete", "trace", "debug" ]: self.output.error("Invalid HTTP method!") exit(1) self.httpmethod = self.arguments.httpmethod.lower() if self.arguments.saveHome: savePath = self.getSavePath() if not FileUtils.exists(savePath): FileUtils.createDirectory(savePath) if FileUtils.exists(savePath) and not FileUtils.isDir(savePath): self.output.error( "Cannot use {} because is a file. Should be a directory". format(savePath)) exit(1) if not FileUtils.canWrite(savePath): self.output.error( "Directory {} is not writable".format(savePath)) exit(1) logs = FileUtils.buildPath(savePath, "logs") if not FileUtils.exists(logs): FileUtils.createDirectory(logs) reports = FileUtils.buildPath(savePath, "reports") if not FileUtils.exists(reports): FileUtils.createDirectory(reports) self.savePath = savePath self.reportsPath = FileUtils.buildPath(self.savePath, "logs") self.blacklists = self.getBlacklists() self.includeStatusCodes = self.arguments.includeStatusCodes self.excludeStatusCodes = self.arguments.excludeStatusCodes self.excludeTexts = self.arguments.excludeTexts self.excludeRegexps = self.arguments.excludeRegexps self.recursive = self.arguments.recursive self.suppressEmpty = self.arguments.suppressEmpty self.minimumResponseSize = self.arguments.minimumResponseSize self.maximumResponseSize = self.arguments.maximumResponseSize self.directories = Queue() self.excludeSubdirs = (arguments.excludeSubdirs if arguments.excludeSubdirs else []) self.dictionary = Dictionary( self.arguments.wordlist, self.arguments.extensions, self.arguments.suffixes, self.arguments.prefixes, self.arguments.lowercase, self.arguments.uppercase, self.arguments.capitalization, self.arguments.forceExtensions, self.arguments.noDotExtensions, self.arguments.excludeExtensions, self.arguments.noExtension) self.errorLog = None self.errorLogPath = None self.threadsLock = Lock() self.batch = False self.batchSession = None self.currentJob = 0 self.allJobs = 0 self.output.header(program_banner) self.printConfig() self.setupErrorLogs() self.output.errorLogFile(self.errorLogPath) if self.arguments.autoSave and len(self.arguments.urlList) > 1: self.setupBatchReports() self.output.newLine("\nAutoSave path: {0}".format( self.batchDirectoryPath)) if self.arguments.useRandomAgents: self.randomAgents = FileUtils.getLines( FileUtils.buildPath(script_path, "db", "user-agents.txt")) try: for url in list(dict.fromkeys(self.arguments.urlList)): try: gc.collect() self.reportManager = ReportManager() self.currentUrl = url self.output.setTarget(self.currentUrl) try: self.requester = Requester( url, cookie=self.arguments.cookie, useragent=self.arguments.useragent, maxPool=self.arguments.threadsCount, maxRetries=self.arguments.maxRetries, delay=self.arguments.delay, timeout=self.arguments.timeout, ip=self.arguments.ip, proxy=self.arguments.proxy, proxylist=self.arguments.proxylist, redirect=self.arguments.redirect, requestByHostname=self.arguments.requestByHostname, httpmethod=self.httpmethod, data=self.arguments.data, ) self.requester.request("") except RequestException as e: self.output.error(e.args[0]["message"]) raise SkipTargetInterrupt if self.arguments.useRandomAgents: self.requester.setRandomAgents(self.randomAgents) for key, value in arguments.headers.items(): self.requester.setHeader(key, value) # Initialize directories Queue with start Path self.basePath = self.requester.basePath if self.arguments.scanSubdirs: for subdir in self.arguments.scanSubdirs: self.directories.put(subdir) self.allJobs += 1 else: self.directories.put("") self.allJobs += 1 self.setupReports(self.requester) matchCallbacks = [self.matchCallback] notFoundCallbacks = [self.notFoundCallback] errorCallbacks = [self.errorCallback, self.appendErrorLog] self.fuzzer = Fuzzer( self.requester, self.dictionary, testFailPath=self.arguments.testFailPath, threads=self.arguments.threadsCount, matchCallbacks=matchCallbacks, notFoundCallbacks=notFoundCallbacks, errorCallbacks=errorCallbacks, ) try: self.wait() except RequestException as e: self.output.error( "Fatal error during site scanning: " + e.args[0]["message"]) raise SkipTargetInterrupt except SkipTargetInterrupt: continue except KeyboardInterrupt: self.output.error("\nCanceled by the user") exit(0) finally: if not self.errorLog.closed: self.errorLog.close() self.reportManager.close() self.output.warning("\nTask Completed")
def setupBatchReports(self): self.batch = True self.batchSession = "BATCH-{0}".format( time.strftime("%y-%m-%d_%H-%M-%S")) self.batchDirectoryPath = FileUtils.buildPath(self.savePath, "reports", self.batchSession) if not FileUtils.exists(self.batchDirectoryPath): FileUtils.createDirectory(self.batchDirectoryPath) if not FileUtils.exists(self.batchDirectoryPath): self.output.error("Couldn't create batch folder {}".format( self.batchDirectoryPath)) sys.exit(1) if FileUtils.canWrite(self.batchDirectoryPath): FileUtils.createDirectory(self.batchDirectoryPath) targetsFile = FileUtils.buildPath(self.batchDirectoryPath, "TARGETS.txt") FileUtils.writeLines(targetsFile, self.arguments.urlList) else: self.output.error("Couldn't create batch folder {}.".format( self.batchDirectoryPath)) sys.exit(1)
def setupReports(self, requester): if self.arguments.autoSave: basePath = "/" if not (len( requester.basePath)) else requester.basePath basePath = basePath.replace(os.path.sep, ".")[:-1] fileName = None directoryPath = None if self.batch: fileName = requester.host directoryPath = self.batchDirectoryPath else: fileName = ('{}_'.format(basePath)) fileName += time.strftime('%y-%m-%d_%H-%M-%S.txt') directoryPath = FileUtils.buildPath(self.savePath, 'reports', requester.host) outputFile = FileUtils.buildPath(directoryPath, fileName) self.output.outputFile(outputFile) if FileUtils.exists(outputFile): i = 2 while FileUtils.exists(outputFile + "_" + str(i)): i += 1 outputFile += "_" + str(i) if not FileUtils.exists(directoryPath): FileUtils.createDirectory(directoryPath) if not FileUtils.exists(directoryPath): self.output.error( "Couldn't create reports folder {}".format( directoryPath)) sys.exit(1) if FileUtils.canWrite(directoryPath): report = None if self.arguments.autoSaveFormat == "simple": report = SimpleReport(requester.host, requester.port, requester.protocol, requester.basePath, outputFile, self.batch) if self.arguments.autoSaveFormat == "json": report = JSONReport( requester.host, requester.port, requester.protocol, requester.basePath, outputFile, ) else: report = PlainTextReport(requester.host, requester.port, requester.protocol, requester.basePath, outputFile, self.batch) self.reportManager.addOutput(report) else: self.output.error( "Can't write reports to {}".format(directoryPath)) sys.exit(1) # TODO: format, refactor code if self.arguments.simpleOutputFile: self.reportManager.addOutput( SimpleReport(requester.host, requester.port, requester.protocol, requester.basePath, self.arguments.simpleOutputFile, self.batch)) if self.arguments.plainTextOutputFile: self.reportManager.addOutput( PlainTextReport(requester.host, requester.port, requester.protocol, requester.basePath, self.arguments.plainTextOutputFile, self.batch)) if self.arguments.jsonOutputFile: self.reportManager.addOutput( JSONReport(requester.host, requester.port, requester.protocol, requester.basePath, self.arguments.jsonOutputFile, self.batch))
def setupErrorLogs(self): fileName = "errors-{0}.log".format(time.strftime("%y-%m-%d_%H-%M-%S")) self.errorLogPath = FileUtils.buildPath(FileUtils.buildPath(self.savePath, "logs", fileName)) self.errorLog = open(self.errorLogPath, "w")
def setupErrorLogs(self): fileName = "errors-{0}.log".format(time.strftime("%y-%m-%d_%H-%M-%S")) self.errorLogPath = FileUtils.buildPath( FileUtils.buildPath(self.savePath, "logs", fileName)) self.errorLog = open(self.errorLogPath, "w")
def __init__(self, script_path,config): logger.add('runtime.log') default_headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", "Accept-Language": "*", "Accept-Encoding": "*", "Keep-Alive": "300", "Cache-Control": "max-age=0", } self.script_path = script_path self.save_path = script_path self.config = config if self.config.httpmethod.lower() not in ["get", "head", "post", "put", "patch", "options", "delete", "trace", "debug"]: logger.debug("Invalid http method!") exit(1) self.includeStatusCodes = self.config.includeStatusCodes self.excludeStatusCodes = self.config.excludeStatusCodes self.excludeTexts = self.config.excludeTexts self.excludeRegexps = self.config.excludeRegexps self.httpmethod = self.config.httpmethod.lower() #self.dicpath = (FileUtils.buildPath(self.script_path,self.config.dicpath)) self.Readdictionary = Dictionary(self.config.dicpath, self.config.extensions, self.config.suffixes, self.config.prefixes, self.config.lowercase, self.config.uppercase, self.config.forceExtensions, self.config.noDotExtensions, self.config.excludeExtensions) self.dictionary =self.Readdictionary.generate() #print(self.dictionary) self.urlList = FileUtils.getLines( FileUtils.buildPath(self.script_path, "target.txt") ) self.scanresult = [] self.reqList = {}#存储self.requester self.scannerList = {}#存储self.scanners self.fuzzList = {} scanFlag = True badUrl = [] if self.config.useRandomAgents: self.randomAgents = FileUtils.getLines( FileUtils.buildPath(self.script_path, "db", "user-agents.txt") ) logger.debug("[+]check urlList.超时的会移出扫描列表") for currentdic in self.dictionary: # print(currentdic) for url in self.urlList: try: if scanFlag: self.requester = Requester( url, cookie=self.config.cookie, useragent=self.config.useragent, maxPool=self.config.threadsCount, maxRetries=self.config.maxRetries, delay=self.config.delay, timeout=self.config.timeout, ip=self.config.ip, proxy=self.config.proxy, proxylist=self.config.proxylist, redirect=self.config.redirect, requestByHostname=self.config.requestByHostname, httpmethod=self.config.httpmethod, data=self.config.data, ) self.requester.request("/") self.reqList[url] = self.requester matchCallbacks = [self.matchCallback] notFoundCallbacks = [self.notFoundCallback] errorCallbacks = [self.errorCallback, self.appendErrorLog] self.fuzzer = Fuzzer( self.requester, self.dictionary, self.config, testFailPath=self.config.testFailPath, threads=self.config.threadsCount, matchCallbacks=matchCallbacks, notFoundCallbacks=notFoundCallbacks, errorCallbacks=errorCallbacks, ) self.fuzzer.setupScanners() self.fuzzList[url] = self.fuzzer #self.scannerList[url]=self.fuzzer.setupScanners() else: # print(self.reqList) self.requester =self.reqList[url] self.fuzzer = self.fuzzList[url] #self.scannerList[url]=self.fuzzer.setupScanners() #logger.debug("[+]scan:%s %s"%(url,currentdic)) self.fuzzer.start(currentdic) except: logger.debug("[-]Error:%s timeout"%(url)) badUrl.append(url) for bad in badUrl: self.urlList.remove(bad) badUrl=[] scanFlag = False if self.config.useRandomAgents: self.requester.setRandomAgents(self.randomAgents)
def __init__(self, script_path, arguments, output): global VERSION program_banner = open( FileUtils.buildPath(script_path, "lib", "controller", "banner.txt")).read().format(**VERSION) self.script_path = script_path self.exit = False self.arguments = arguments self.output = output self.savePath = self.script_path if self.arguments.saveHome: savePath = self.getSavePath() if not FileUtils.exists(savePath): FileUtils.createDirectory(savePath) if FileUtils.exists(savePath) and not FileUtils.isDir(savePath): self.output.error( 'Cannot use {} because is a file. Should be a directory'. format(savePath)) exit(1) if not FileUtils.canWrite(savePath): self.output.error( 'Directory {} is not writable'.format(savePath)) exit(1) logs = FileUtils.buildPath(savePath, "logs") if not FileUtils.exists(logs): FileUtils.createDirectory(logs) reports = FileUtils.buildPath(savePath, "reports") if not FileUtils.exists(reports): FileUtils.createDirectory(reports) self.savePath = savePath self.reportsPath = FileUtils.buildPath(self.savePath, "logs") self.blacklists = self.getBlacklists() self.fuzzer = None self.excludeStatusCodes = self.arguments.excludeStatusCodes self.recursive = self.arguments.recursive self.suppressEmpty = self.arguments.suppressEmpty self.directories = Queue() self.excludeSubdirs = (arguments.excludeSubdirs if arguments.excludeSubdirs is not None else []) self.output.header(program_banner) self.dictionary = Dictionary(self.arguments.wordlist, self.arguments.extensions, self.arguments.lowercase, self.arguments.forceExtensions) self.printConfig() self.errorLog = None self.errorLogPath = None self.errorLogLock = Lock() self.batch = False self.batchSession = None self.setupErrorLogs() self.output.newLine("\nError Log: {0}".format(self.errorLogPath)) if self.arguments.autoSave and len(self.arguments.urlList) > 1: self.setupBatchReports() self.output.newLine("\nAutoSave path: {0}".format( self.batchDirectoryPath)) if self.arguments.useRandomAgents: self.randomAgents = FileUtils.getLines( FileUtils.buildPath(script_path, "db", "user-agents.txt")) try: for url in self.arguments.urlList: try: gc.collect() self.reportManager = ReportManager() self.currentUrl = url self.output.target(self.currentUrl) try: self.requester = Requester( url, cookie=self.arguments.cookie, useragent=self.arguments.useragent, maxPool=self.arguments.threadsCount, maxRetries=self.arguments.maxRetries, delay=self.arguments.delay, timeout=self.arguments.timeout, ip=self.arguments.ip, proxy=self.arguments.proxy, redirect=self.arguments.redirect, requestByHostname=self.arguments.requestByHostname) self.requester.request("/") except RequestException as e: self.output.error(e.args[0]['message']) raise SkipTargetInterrupt if self.arguments.useRandomAgents: self.requester.setRandomAgents(self.randomAgents) for key, value in arguments.headers.items(): self.requester.setHeader(key, value) # Initialize directories Queue with start Path self.basePath = self.requester.basePath if self.arguments.scanSubdirs is not None: for subdir in self.arguments.scanSubdirs: self.directories.put(subdir) else: self.directories.put('') self.setupReports(self.requester) matchCallbacks = [self.matchCallback] notFoundCallbacks = [self.notFoundCallback] errorCallbacks = [self.errorCallback, self.appendErrorLog] self.fuzzer = Fuzzer( self.requester, self.dictionary, testFailPath=self.arguments.testFailPath, threads=self.arguments.threadsCount, matchCallbacks=matchCallbacks, notFoundCallbacks=notFoundCallbacks, errorCallbacks=errorCallbacks) try: self.wait() except RequestException as e: self.output.error( "Fatal error during site scanning: " + e.args[0]['message']) raise SkipTargetInterrupt except SkipTargetInterrupt: continue finally: self.reportManager.save() except KeyboardInterrupt: self.output.error('\nCanceled by the user') exit(0) finally: if not self.errorLog.closed: self.errorLog.close() self.reportManager.close() self.output.warning('\nTask Completed')
def __init__(self, script_path, arguments, output): global VERSION program_banner = (open( FileUtils.build_path(script_path, "lib", "controller", "banner.txt")).read().format(**VERSION)) self.directories = Queue() self.script_path = script_path self.exit = False self.arguments = arguments self.output = output self.savePath = self.script_path self.doneDirs = [] if arguments.raw_file: # Overwrite python-requests default headers default_headers = { "User-Agent": None, "Accept-Encoding": None, "Accept": None, } _raw = Raw(arguments.raw_file, arguments.scheme) self.urlList = [_raw.url()] self.httpmethod = _raw.method() self.data = _raw.data() self.headers = {**default_headers, **_raw.headers()} self.cookie = _raw.cookie() self.useragent = _raw.user_agent() else: default_headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", "Accept-Language": "*", "Accept-Encoding": "*", "Keep-Alive": "300", "Cache-Control": "max-age=0", } self.urlList = list(filter(None, dict.fromkeys(arguments.urlList))) self.httpmethod = arguments.httpmethod.lower() self.data = arguments.data self.headers = {**default_headers, **arguments.headers} self.cookie = arguments.cookie self.useragent = arguments.useragent self.recursion_depth = arguments.recursion_depth if arguments.saveHome: savePath = self.getSavePath() if not FileUtils.exists(savePath): FileUtils.create_directory(savePath) if FileUtils.exists(savePath) and not FileUtils.is_dir(savePath): self.output.error( "Cannot use {} because it's a file. Should be a directory". format(savePath)) exit(1) if not FileUtils.can_write(savePath): self.output.error( "Directory {} is not writable".format(savePath)) exit(1) logs = FileUtils.build_path(savePath, "logs") if not FileUtils.exists(logs): FileUtils.create_directory(logs) reports = FileUtils.build_path(savePath, "reports") if not FileUtils.exists(reports): FileUtils.create_directory(reports) self.savePath = savePath self.reportsPath = FileUtils.build_path(self.savePath, "logs") self.blacklists = self.getBlacklists() self.includeStatusCodes = arguments.includeStatusCodes self.excludeStatusCodes = arguments.excludeStatusCodes self.excludeSizes = arguments.excludeSizes self.excludeTexts = arguments.excludeTexts self.excludeRegexps = arguments.excludeRegexps self.excludeRedirects = arguments.excludeRedirects self.recursive = arguments.recursive self.minimumResponseSize = arguments.minimumResponseSize self.maximumResponseSize = arguments.maximumResponseSize self.scanSubdirs = arguments.scanSubdirs self.excludeSubdirs = (arguments.excludeSubdirs if arguments.excludeSubdirs else []) self.dictionary = Dictionary( arguments.wordlist, arguments.extensions, arguments.suffixes, arguments.prefixes, arguments.lowercase, arguments.uppercase, arguments.capitalization, arguments.forceExtensions, arguments.excludeExtensions, arguments.noExtension, arguments.onlySelected) self.allJobs = len(self.scanSubdirs) if self.scanSubdirs else 1 self.currentJob = 0 self.errorLog = None self.errorLogPath = None self.threadsLock = Lock() self.batch = False self.batchSession = None self.skip429 = False self.output.header(program_banner) self.printConfig() self.setupErrorLogs() self.output.errorLogFile(self.errorLogPath) if arguments.autoSave and len(self.urlList) > 1: self.setupBatchReports() self.output.newLine("\nAutoSave path: {0}".format( self.batchDirectoryPath)) if arguments.useRandomAgents: self.randomAgents = FileUtils.get_lines( FileUtils.build_path(script_path, "db", "user-agents.txt")) try: for url in self.urlList: try: gc.collect() self.reportManager = ReportManager() self.currentUrl = url if url.endswith("/") else url + "/" self.output.setTarget(self.currentUrl, self.arguments.scheme) try: self.requester = Requester( url, cookie=self.cookie, useragent=self.useragent, maxPool=arguments.threadsCount, maxRetries=arguments.maxRetries, timeout=arguments.timeout, ip=arguments.ip, proxy=arguments.proxy, proxylist=arguments.proxylist, redirect=arguments.redirect, requestByHostname=arguments.requestByHostname, httpmethod=self.httpmethod, data=self.data, scheme=arguments.scheme, ) for key, value in self.headers.items(): self.requester.setHeader(key, value) self.requester.request("") except RequestException as e: self.output.error(e.args[0]["message"]) raise SkipTargetInterrupt if arguments.useRandomAgents: self.requester.setRandomAgents(self.randomAgents) # Initialize directories Queue with start Path self.basePath = self.requester.basePath if self.scanSubdirs: for subdir in self.scanSubdirs: self.directories.put(subdir) else: self.directories.put("") self.setupReports(self.requester) matchCallbacks = [self.matchCallback] notFoundCallbacks = [self.notFoundCallback] errorCallbacks = [self.errorCallback, self.appendErrorLog] self.fuzzer = Fuzzer( self.requester, self.dictionary, testFailPath=arguments.testFailPath, threads=arguments.threadsCount, delay=arguments.delay, matchCallbacks=matchCallbacks, notFoundCallbacks=notFoundCallbacks, errorCallbacks=errorCallbacks, ) try: self.prepare() except RequestException as e: self.output.error("Fatal error during scanning: " + e.args[0]["message"]) raise SkipTargetInterrupt except SkipTargetInterrupt: continue except KeyboardInterrupt: self.output.error("\nCanceled by the user") exit(0) finally: if not self.errorLog.closed: self.errorLog.close() self.reportManager.close() self.output.warning("\nTask Completed")
def matchCallback(self, path): self.index += 1 if self.arguments.skip_on_429 and path.status == 429: self.skip429 = True return if (path.status and path.status not in self.excludeStatusCodes) and ( not self.includeStatusCodes or path.status in self.includeStatusCodes ) and (not self.blacklists.get(path.status) or path.path not in self.blacklists.get(path.status)) and ( not self.excludeSizes or FileUtils.size_human(len(path.response.body)).strip() not in self.excludeSizes) and ( not self.minimumResponseSize or self.minimumResponseSize < len(path.response.body) ) and (not self.maximumResponseSize or self.maximumResponseSize > len(path.response.body)): for excludeText in self.excludeTexts: if excludeText in path.response.body.decode('iso8859-1'): del path return for excludeRegexp in self.excludeRegexps: if (re.search(excludeRegexp, path.response.body.decode('iso8859-1')) is not None): del path return for excludeRedirect in self.excludeRedirects: if path.response.redirect and ( (re.match(excludeRedirect, path.response.redirect.decode('iso8859-1')) is not None) or (excludeRedirect in path.response.redirect)): del path return addedToQueue = False if self.recursive and "?" not in path.path and "#" not in path.path: if path.response.redirect: addedToQueue = self.addRedirectDirectory(path) else: addedToQueue = self.addDirectory(path.path) self.output.statusReport(path.path, path.response, self.arguments.full_url, addedToQueue) if self.arguments.replay_proxy: self.requester.request(path.path, proxy=self.arguments.replay_proxy) newPath = self.currentDirectory + path.path self.reportManager.addPath(newPath, path.status, path.response) self.reportManager.save() del path
def __init__(self, script_path, arguments, output): global VERSION program_banner = open( FileUtils.buildPath(script_path, "lib", "controller", "banner.txt")).read().format(**VERSION) self.script_path = script_path self.exit = False self.arguments = arguments self.output = output self.savePath = self.script_path self.doneDirs = [] self.recursive_level_max = self.arguments.recursive_level_max if self.arguments.httpmethod.lower() not in ["get", "head", "post"]: self.output.error("Inavlid http method!") exit(1) self.httpmethod = self.arguments.httpmethod.lower() if self.arguments.saveHome: savePath = self.getSavePath() if not FileUtils.exists(savePath): FileUtils.createDirectory(savePath) if FileUtils.exists(savePath) and not FileUtils.isDir(savePath): self.output.error( 'Cannot use {} because is a file. Should be a directory'. format(savePath)) exit(1) if not FileUtils.canWrite(savePath): self.output.error( 'Directory {} is not writable'.format(savePath)) exit(1) logs = FileUtils.buildPath(savePath, "logs") if not FileUtils.exists(logs): FileUtils.createDirectory(logs) reports = FileUtils.buildPath(savePath, "reports") if not FileUtils.exists(reports): FileUtils.createDirectory(reports) self.savePath = savePath self.reportsPath = FileUtils.buildPath(self.savePath, "logs") self.blacklists = self.getBlacklists() self.blacklists = {} self.fuzzer = None self.excludeStatusCodes = self.arguments.excludeStatusCodes self.excludeTexts = self.arguments.excludeTexts self.excludeRegexps = self.arguments.excludeRegexps self.recursive = self.arguments.recursive self.suppressEmpty = self.arguments.suppressEmpty self.directories = Queue() self.excludeSubdirs = (arguments.excludeSubdirs if arguments.excludeSubdirs is not None else []) self.output.header(program_banner) # self.dictionary = Dictionary(self.arguments.wordlist, self.arguments.extensions, # self.arguments.lowercase, self.arguments.forceExtensions) # self.printConfig() self.errorLog = None self.errorLogPath = None self.errorLogLock = Lock() self.batch = False self.batchSession = None self.setupErrorLogs() self.output.newLine("\nError Log: {0}".format(self.errorLogPath)) if self.arguments.autoSave and len(self.arguments.urlList) > 1: self.setupBatchReports() self.output.newLine("\nAutoSave path: {0}".format( self.batchDirectoryPath)) if self.arguments.useRandomAgents: self.randomAgents = FileUtils.getLines( FileUtils.buildPath(script_path, "db", "user-agents.txt")) try: for url in self.arguments.urlList: try: gc.collect() self.reportManager = ReportManager() self.currentUrl = url self.output.target(self.currentUrl) try: # DNS A Record query self.requester = Requester( url, script_path=self.script_path, cookie=self.arguments.cookie, useragent=self.arguments.useragent, maxPool=self.arguments.threadsCount, maxRetries=self.arguments.maxRetries, delay=self.arguments.delay, timeout=self.arguments.timeout, ip=self.arguments.ip, proxy=self.arguments.proxy, redirect=self.arguments.redirect, requestByHostname=self.arguments.requestByHostname, httpmethod=self.httpmethod) # 网站连通性测试 site_connection_test_resp = self.requester.request( self.requester.basePath, use_base_path=False, allow_redirect=True, fingerprint=True) self.dictionary = Dictionary(self.requester.scan_list, self.requester.directory, self.requester.filename, self.requester.extension) # 404 page if self.requester.url_type == URLType.normal_restful_dir: path_404 = '{}/{}/'.format( self.requester.basePath, RandomUtils.randString(8)) path_404 = path_404.replace("//", "/") elif self.requester.url_type == URLType.restful_file: path_404 = self.requester.basePath.replace( self.requester.filename, RandomUtils.randString( len(self.requester.filename) or 8)) elif self.requester.url_type == URLType.normal_file: path_404 = self.requester.basePath.replace( self.requester.filename, RandomUtils.randString( len(self.requester.filename) or 8)) path_404_quote = self.dictionary.quote(path_404) response_404 = self.requester.request( path_404_quote, use_base_path=False, allow_redirect=False) # Waf 探测 waf_exist, waf_response = self.requester.waf_detect( site_connection_test_resp.body, url_quote=self.dictionary.quote) except RequestException as e: self.output.error(e.args[0]['message']) raise SkipTargetInterrupt if self.arguments.useRandomAgents: self.requester.setRandomAgents(self.randomAgents) for key, value in arguments.headers.items(): self.requester.setHeader(key, value) # Initialize directories Queue with start Path self.basePath = self.requester.basePath if self.arguments.scanSubdirs is not None: for subdir in self.arguments.scanSubdirs: self.directories.put(subdir) else: self.directories.put('') self.setupReports(self.requester) matchCallbacks = [self.matchCallback] notFoundCallbacks = [self.notFoundCallback] errorCallbacks = [self.errorCallback, self.appendErrorLog] self.fuzzer = Fuzzer( self.requester, self.dictionary, waf_exist, waf_response, response_404, testFailPath=self.arguments.testFailPath, threads=self.arguments.threadsCount, matchCallbacks=matchCallbacks, notFoundCallbacks=notFoundCallbacks, errorCallbacks=errorCallbacks) try: self.wait() except RequestException as e: self.output.error( "Fatal error during site scanning: " + e.args[0]['message']) raise SkipTargetInterrupt except SkipTargetInterrupt: continue finally: self.reportManager.save() except KeyboardInterrupt: self.output.error('\nCanceled by the user') exit(0) finally: if not self.errorLog.closed: self.errorLog.close() self.reportManager.close() self.output.warning('\nTask Completed')
def setupReports(self, requester): if self.arguments.autoSave: basePath = "/" if requester.basePath is "" else requester.basePath basePath = basePath.replace(os.path.sep, ".")[1:-1] fileName = None directoryPath = None if self.batch: fileName = requester.host directoryPath = self.batchDirectoryPath else: fileName = "{}_".format(basePath) if basePath is not "" else "" fileName += time.strftime("%y-%m-%d_%H-%M-%S") directoryPath = FileUtils.buildPath(self.savePath, "reports", requester.host) outputFile = FileUtils.buildPath(directoryPath, fileName) if FileUtils.exists(outputFile): i = 2 while FileUtils.exists(outputFile + "_" + str(i)): i += 1 outputFile += "_" + str(i) if not FileUtils.exists(directoryPath): FileUtils.createDirectory(directoryPath) if not FileUtils.exists(directoryPath): self.output.error("Couldn't create reports folder {}".format(directoryPath)) sys.exit(1) if FileUtils.canWrite(directoryPath): report = None if self.arguments.autoSaveFormat == "simple": report = SimpleReport( requester.host, requester.port, requester.protocol, requester.basePath, outputFile ) if self.arguments.autoSaveFormat == "json": report = JSONReport( requester.host, requester.port, requester.protocol, requester.basePath, outputFile ) else: report = PlainTextReport( requester.host, requester.port, requester.protocol, requester.basePath, outputFile ) self.reportManager.addOutput(report) else: self.output.error("Can't write reports to {}".format(directoryPath)) sys.exit(1) if self.arguments.simpleOutputFile is not None: self.reportManager.addOutput( SimpleReport( requester.host, requester.port, requester.protocol, requester.basePath, self.arguments.simpleOutputFile, ) ) if self.arguments.plainTextOutputFile is not None: self.reportManager.addOutput( PlainTextReport( requester.host, requester.port, requester.protocol, requester.basePath, self.arguments.plainTextOutputFile, ) ) if self.arguments.jsonOutputFile is not None: self.reportManager.addOutput( JSONReport( requester.host, requester.port, requester.protocol, requester.basePath, self.arguments.jsonOutputFile, ) )
def __init__(self, script_path, arguments, output): global VERSION program_banner = (open( FileUtils.build_path(script_path, "lib", "controller", "banner.txt")).read().format(**VERSION)) self.directories = Queue() self.script_path = script_path self.exit = False self.arguments = arguments self.output = output self.done_dirs = [] if arguments.raw_file: _raw = Raw(arguments.raw_file, arguments.scheme) self.url_list = [_raw.url()] self.httpmethod = _raw.method() self.data = _raw.data() self.headers = _raw.headers() else: default_headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", "Accept-Language": "*", "Accept-Encoding": "*", "Keep-Alive": "300", "Cache-Control": "max-age=0", } self.url_list = list( filter(None, dict.fromkeys(arguments.url_list))) self.httpmethod = arguments.httpmethod.lower() self.data = arguments.data self.headers = {**default_headers, **arguments.headers} if arguments.cookie: self.headers["Cookie"] = arguments.cookie if arguments.useragent: self.headers["User-Agent"] = arguments.useragent self.recursion_depth = arguments.recursion_depth if arguments.logs_location and self.validate_path( arguments.logs_location): self.logs_path = FileUtils.build_path(arguments.logs_location) elif self.validate_path(self.script_path): self.logs_path = FileUtils.build_path(self.script_path, "logs") if not FileUtils.exists(self.logs_path): FileUtils.create_directory(self.logs_path) if arguments.output_location and self.validate_path( arguments.output_location): self.save_path = FileUtils.build_path(arguments.output_location) elif self.validate_path(self.script_path): self.save_path = FileUtils.build_path(self.script_path, "reports") if not FileUtils.exists(self.save_path): FileUtils.create_directory(self.save_path) self.blacklists = self.get_blacklists() self.include_status_codes = arguments.include_status_codes self.exclude_status_codes = arguments.exclude_status_codes self.exclude_sizes = arguments.exclude_sizes self.exclude_texts = arguments.exclude_texts self.exclude_regexps = arguments.exclude_regexps self.exclude_redirects = arguments.exclude_redirects self.recursive = arguments.recursive self.deep_recursive = arguments.deep_recursive self.force_recursive = arguments.force_recursive self.recursion_status_codes = arguments.recursion_status_codes self.minimum_response_size = arguments.minimum_response_size self.maximum_response_size = arguments.maximum_response_size self.maxtime = arguments.maxtime self.scan_subdirs = arguments.scan_subdirs self.exclude_subdirs = arguments.exclude_subdirs self.dictionary = Dictionary( paths=arguments.wordlist, extensions=arguments.extensions, suffixes=arguments.suffixes, prefixes=arguments.prefixes, lowercase=arguments.lowercase, uppercase=arguments.uppercase, capitalization=arguments.capitalization, forced_extensions=arguments.force_extensions, exclude_extensions=arguments.exclude_extensions, no_extension=arguments.no_extension, only_selected=arguments.only_selected) self.all_jobs = len(self.scan_subdirs) if self.scan_subdirs else 1 self.current_job = 0 self.start_time = time.time() self.error_log = None self.error_log_path = None self.threads_lock = Lock() self.batch = False self.batch_session = None self.output.header(program_banner) self.print_config() if arguments.use_random_agents: self.random_agents = FileUtils.get_lines( FileUtils.build_path(script_path, "db", "user-agents.txt")) self.report_manager = EmptyReportManager() self.report = EmptyReport() if arguments.autosave_report or arguments.output_file: if len(self.url_list) > 1: self.setup_batch_reports() self.setup_reports() self.setup_error_logs() self.output.error_log_file(self.error_log_path) try: for url in self.url_list: try: gc.collect() url = url if url.endswith("/") else url + "/" self.output.set_target(url, self.arguments.scheme) try: self.requester = Requester( url, max_pool=arguments.threads_count, max_retries=arguments.max_retries, timeout=arguments.timeout, ip=arguments.ip, proxy=arguments.proxy, proxylist=arguments.proxylist, redirect=arguments.redirect, request_by_hostname=arguments.request_by_hostname, httpmethod=self.httpmethod, data=self.data, scheme=arguments.scheme, ) for key, value in self.headers.items(): self.requester.set_header(key, value) if arguments.auth: self.requester.set_auth(arguments.auth_type, arguments.auth) self.requester.request("") if arguments.autosave_report or arguments.output_file: self.report = Report(self.requester.host, self.requester.port, self.requester.protocol, self.requester.base_path) except RequestException as e: self.output.error(e.args[0]["message"]) raise SkipTargetInterrupt if arguments.use_random_agents: self.requester.set_random_agents(self.random_agents) # Initialize directories Queue with start Path self.base_path = self.requester.base_path self.status_skip = None for subdir in self.scan_subdirs: self.directories.put(subdir) else: self.directories.put("") match_callbacks = [self.match_callback] not_found_callbacks = [self.not_found_callback] error_callbacks = [ self.error_callback, self.append_error_log ] self.fuzzer = Fuzzer( self.requester, self.dictionary, suffixes=arguments.suffixes, prefixes=arguments.prefixes, exclude_content=arguments.exclude_content, threads=arguments.threads_count, delay=arguments.delay, maxrate=arguments.maxrate, match_callbacks=match_callbacks, not_found_callbacks=not_found_callbacks, error_callbacks=error_callbacks, ) try: self.prepare() except RequestException as e: self.output.error(e.args[0]["message"]) raise SkipTargetInterrupt except SkipTargetInterrupt: self.report.completed = True continue except KeyboardInterrupt: self.output.error("\nCanceled by the user") exit(0) finally: if not self.error_log.closed: self.error_log.close() self.output.warning("\nTask Completed")
def setupErrorLogs(self): fileName = "errors-{0}.log".format(time.strftime('%y-%m-%d_%H-%M-%S')) self.errorLogPath = FileUtils.buildPath(self.script_path, 'logs', fileName) self.errorLog = open(self.errorLogPath, "w")
def setupBatchReports(self): self.batch = True self.batchSession = "BATCH-{0}".format(time.strftime('%y-%m-%d_%H-%M-%S')) self.batchDirectoryPath = FileUtils.buildPath(self.script_path, 'reports', self.batchSession) if not FileUtils.exists(self.batchDirectoryPath): FileUtils.createDirectory(self.batchDirectoryPath) if not FileUtils.exists(self.batchDirectoryPath): self.output.error("Couldn't create batch folder {}".format(self.batchDirectoryPath)) sys.exit(1) if FileUtils.canWrite(self.batchDirectoryPath): FileUtils.createDirectory(self.batchDirectoryPath) targetsFile = FileUtils.buildPath(self.batchDirectoryPath, "TARGETS.txt") FileUtils.writeLines(targetsFile, self.arguments.urlList) else: self.output.error("Couldn't create batch folder {}.".format(self.batchDirectoryPath)) sys.exit(1)
def __init__(self, script_path, arguments, output): global VERSION PROGRAM_BANNER = open(FileUtils.buildPath(script_path, "lib", "controller", "banner.txt")).read().format( **VERSION) self.script_path = script_path self.exit = False self.arguments = arguments self.output = output self.blacklists = self.getBlacklists() self.fuzzer = None self.excludeStatusCodes = self.arguments.excludeStatusCodes self.recursive = self.arguments.recursive self.directories = Queue() self.excludeSubdirs = (arguments.excludeSubdirs if arguments.excludeSubdirs is not None else []) self.output.header(PROGRAM_BANNER) self.dictionary = Dictionary(self.arguments.wordlist, self.arguments.extensions, self.arguments.lowercase) self.printConfig() self.errorLog = None self.errorLogPath = None self.errorLogLock = Lock() self.batch = False self.batchSession = None self.setupErrorLogs() self.output.newLine("\nError Log: {0}".format(self.errorLogPath)) if self.arguments.autoSave and len(self.arguments.urlList) > 1: self.setupBatchReports() self.output.newLine("\nAutoSave path: {0}".format(self.batchDirectoryPath)) if self.arguments.useRandomAgents: self.randomAgents = FileUtils.getLines(FileUtils.buildPath(script_path, "db", "user-agents.txt")) try: for url in self.arguments.urlList: try: gc.collect() self.reportManager = ReportManager() self.currentUrl = url self.requester = Requester(url, cookie=self.arguments.cookie, useragent=self.arguments.useragent, maxPool=self.arguments.threadsCount, maxRetries=self.arguments.maxRetries, timeout=self.arguments.timeout, ip=self.arguments.ip, proxy=self.arguments.proxy, redirect=self.arguments.redirect) if self.arguments.useRandomAgents: self.requester.setRandomAgents(self.randomAgents) for key, value in arguments.headers.items(): self.requester.setHeader(key, value) # Initialize directories Queue with start Path self.basePath = self.requester.basePath if self.arguments.scanSubdirs is not None: for subdir in self.arguments.scanSubdirs: self.directories.put(subdir) else: self.directories.put('') self.setupReports(self.requester) self.output.target(self.currentUrl) matchCallbacks = [self.matchCallback] notFoundCallbacks = [self.notFoundCallback] errorCallbacks = [self.errorCallback, self.appendErrorLog] self.fuzzer = Fuzzer(self.requester, self.dictionary, testFailPath=self.arguments.testFailPath, threads=self.arguments.threadsCount, matchCallbacks=matchCallbacks, notFoundCallbacks=notFoundCallbacks, errorCallbacks=errorCallbacks) self.wait() except SkipTargetInterrupt: continue finally: self.reportManager.save() except KeyboardInterrupt: self.output.error('\nCanceled by the user') exit(0) finally: if not self.errorLog.closed: self.errorLog.close() self.reportManager.close() self.output.warning('\nTask Completed')
def __init__(self, script_path, arguments, output): global VERSION program_banner = open(FileUtils.buildPath(script_path, "lib", "controller", "banner.txt")).read().format( **VERSION) self.script_path = script_path self.exit = False self.arguments = arguments self.output = output self.savePath = self.script_path self.doneDirs = [] self.recursive_level_max = self.arguments.recursive_level_max if self.arguments.httpmethod.lower() not in ["get", "head", "post"]: self.output.error("Inavlid http method!") exit(1) self.httpmethod = self.arguments.httpmethod.lower() if self.arguments.saveHome: savePath = self.getSavePath() if not FileUtils.exists(savePath): FileUtils.createDirectory(savePath) if FileUtils.exists(savePath) and not FileUtils.isDir(savePath): self.output.error('Cannot use {} because is a file. Should be a directory'.format(savePath)) exit(1) if not FileUtils.canWrite(savePath): self.output.error('Directory {} is not writable'.format(savePath)) exit(1) logs = FileUtils.buildPath(savePath, "logs") if not FileUtils.exists(logs): FileUtils.createDirectory(logs) reports = FileUtils.buildPath(savePath, "reports") if not FileUtils.exists(reports): FileUtils.createDirectory(reports) self.savePath = savePath self.reportsPath = FileUtils.buildPath(self.savePath, "logs") self.blacklists = self.getBlacklists() self.fuzzer = None self.excludeStatusCodes = self.arguments.excludeStatusCodes self.recursive = self.arguments.recursive self.suppressEmpty = self.arguments.suppressEmpty self.directories = Queue() self.excludeSubdirs = (arguments.excludeSubdirs if arguments.excludeSubdirs is not None else []) self.output.header(program_banner) self.dictionary = Dictionary(self.arguments.wordlist, self.arguments.extensions, self.arguments.lowercase, self.arguments.forceExtensions) self.printConfig() self.errorLog = None self.errorLogPath = None self.errorLogLock = Lock() self.batch = False self.batchSession = None self.setupErrorLogs() self.output.newLine("\nError Log: {0}".format(self.errorLogPath)) if self.arguments.autoSave and len(self.arguments.urlList) > 1: self.setupBatchReports() self.output.newLine("\nAutoSave path: {0}".format(self.batchDirectoryPath)) if self.arguments.useRandomAgents: self.randomAgents = FileUtils.getLines(FileUtils.buildPath(script_path, "db", "user-agents.txt")) try: for url in self.arguments.urlList: try: gc.collect() self.reportManager = ReportManager() self.currentUrl = url self.output.target(self.currentUrl) try: self.requester = Requester(url, cookie=self.arguments.cookie, useragent=self.arguments.useragent, maxPool=self.arguments.threadsCount, maxRetries=self.arguments.maxRetries, delay=self.arguments.delay, timeout=self.arguments.timeout, ip=self.arguments.ip, proxy=self.arguments.proxy, redirect=self.arguments.redirect, requestByHostname=self.arguments.requestByHostname, httpmethod=self.httpmethod) self.requester.request("/") except RequestException as e: self.output.error(e.args[0]['message']) raise SkipTargetInterrupt if self.arguments.useRandomAgents: self.requester.setRandomAgents(self.randomAgents) for key, value in arguments.headers.items(): self.requester.setHeader(key, value) # Initialize directories Queue with start Path self.basePath = self.requester.basePath if self.arguments.scanSubdirs is not None: for subdir in self.arguments.scanSubdirs: self.directories.put(subdir) else: self.directories.put('') self.setupReports(self.requester) matchCallbacks = [self.matchCallback] notFoundCallbacks = [self.notFoundCallback] errorCallbacks = [self.errorCallback, self.appendErrorLog] self.fuzzer = Fuzzer(self.requester, self.dictionary, testFailPath=self.arguments.testFailPath, threads=self.arguments.threadsCount, matchCallbacks=matchCallbacks, notFoundCallbacks=notFoundCallbacks, errorCallbacks=errorCallbacks) try: self.wait() except RequestException as e: self.output.error("Fatal error during site scanning: " + e.args[0]['message']) raise SkipTargetInterrupt except SkipTargetInterrupt: continue finally: self.reportManager.save() except KeyboardInterrupt: self.output.error('\nCanceled by the user') exit(0) finally: if not self.errorLog.closed: self.errorLog.close() self.reportManager.close() self.output.warning('\nTask Completed')