def setup(self): firstPath = self.preffix + self.testPath + self.suffix firstResponse = self.requester.request(firstPath) self.invalidStatus = firstResponse.status secondPath = self.preffix + RandomUtils.randString( omit=self.testPath) + self.suffix secondResponse = self.requester.request(secondPath) if self.invalidStatus == 404: # Using the response status code is enough :-} return # look for redirects elif firstResponse.status in self.redirectStatusCodes and firstResponse.redirect and secondResponse.redirect: self.redirectRegExp = self.generateRedirectRegExp( firstResponse.redirect, secondResponse.redirect) # Analyze response bodies self.dynamicParser = DynamicContentParser(self.requester, firstPath, firstResponse.body, secondResponse.body) baseRatio = float("{0:.2f}".format( self.dynamicParser.comparisonRatio)) # Rounding to 2 decimals # If response length is small, adjust ratio if len(firstResponse) < 2000: baseRatio -= 0.1 if baseRatio < self.ratio: self.ratio = baseRatio
def setup(self): firstPath = self.testPath + self.suffix firstResponse = self.requester.request(firstPath) self.invalidStatus = firstResponse.status if self.invalidStatus == 404: # Using the response status code is enough :-} return # look for redirects secondPath = RandomUtils.randString(omit=self.testPath) + self.suffix secondResponse = self.requester.request(secondPath) if firstResponse.status in self.redirectStatusCodes and firstResponse.redirect and secondResponse.redirect: self.redirectRegExp = self.generateRedirectRegExp(firstResponse.redirect, secondResponse.redirect) # Analyze response bodies self.dynamicParser = DynamicContentParser(self.requester, firstPath, firstResponse.body, secondResponse.body) baseRatio = float("{0:.2f}".format(self.dynamicParser.comparisonRatio)) # Rounding to 2 decimals # If response length is small, adjust ratio if len(firstResponse) < 2000: baseRatio -= 0.1 if baseRatio < self.ratio: self.ratio = baseRatio
def setup(self): for path in self.testPath: firstPath = RandomUtils.randString() + '/' + path + self.suffix firstResponse = self.requester.request(firstPath) if firstResponse.status not in self.invalidStatus: self.invalidStatus.append(firstResponse.status) if firstResponse.status == 404: # Using the response status code is enough :-} continue # look for redirects secondPath = RandomUtils.randString() + '/' + path + self.suffix secondResponse = self.requester.request(secondPath) if firstResponse.status in self.redirectStatusCodes and firstResponse.redirect and secondResponse.redirect: self.redirectRegExp.append( self.generateRedirectRegExp(firstResponse.redirect, secondResponse.redirect)) # Analyze response bodies dynamicParser = DynamicContentParser(self.requester, firstPath, firstResponse.body, secondResponse.body) baseRatio = float("{0:.2f}".format( dynamicParser.comparisonRatio)) # Rounding to 2 decimals ratio = self.ratio # If response length is small, adjust ratio if len(firstResponse) < 2000: baseRatio -= 0.1 if baseRatio < self.ratio: ratio = baseRatio if self.dynamicParser: flag = 0 for _dynamicParser, __ in self.dynamicParser: _ratio = dynamicParser.compareTo(_dynamicParser.cleanPage) flag += _ratio > ratio if not flag: self.dynamicParser.append((dynamicParser, ratio)) else: self.dynamicParser.append((dynamicParser, ratio))
def get_special_str(self, str): str_list = list(str) str_set_len = len(set(str_list)) for _ in range(100): if str_set_len == 1 and len(str_list) == 1: new_str = RandomUtils.randString(n=1, omit=str_list) elif str_set_len == 1: _ = RandomUtils.randString(n=len(str_list) - str_set_len) new_str = '{}{}'.format(str_list[0], _) else: random.shuffle(str_list) new_str = ''.join(str_list) if new_str == str: continue return new_str else: raise Exception('Special Str: {}'.format(str))
def get_special_path(self, path): special_path = [] if path.endswith('/'): _origin_path, _dir = path[:-1].rsplit('/', 1) else: _origin_path, _last_element = path.rsplit('/', 1) if path.endswith('/'): # unzip exception _origin_path, _dir = path[:-1].rsplit('/', 1) special_dir = self.get_special_str(_dir) _path = '{}/{}/'.format(_origin_path, special_dir) if not _path.startswith('/'): _path = '/' + _path special_path.append(_path) elif '.' in _last_element: _origin_path, _filename_ext = path.rsplit('/', 1) _filename, _ext = _filename_ext.rsplit('.', 1) # special filename if not _filename: _filename = RandomUtils.randString(n=8) special_filename = self.get_special_str(_filename) _path = '{}/{}.{}'.format(_origin_path, special_filename, _ext) if not _path.startswith('/'): _path = '/' + _path special_path.append(_path) # special extension if not _ext: _ext = RandomUtils.randString(n=8) special_extension = self.get_special_str(_ext) _path = '{}/{}.{}'.format(_origin_path, _filename, special_extension) if not _path.startswith('/'): _path = '/' + _path special_path.append(_path) else: _origin_path, _filename = path.rsplit('/', 1) special_filename = self.get_special_str(_filename) _path = '{}/{}'.format(_origin_path, special_filename) if not _path.startswith('/'): _path = '/' + _path special_path.append(_path) return special_path
def __init__(self, requester, testPath=None, suffix=None): if testPath is None or testPath is "": self.testPath = RandomUtils.randString() else: self.testPath = testPath self.suffix = suffix if suffix is not None else "" self.requester = requester self.tester = None self.redirectRegExp = None self.invalidStatus = None self.dynamicParser = None self.ratio = 0.98 self.redirectStatusCodes = [301, 302, 307] self.setup()
def __init__(self, requester, testPath=None, suffix=None): if testPath is None or testPath is "": self.testPath = RandomUtils.randString() else: self.testPath = testPath self.suffix = suffix if suffix is not None else "" self.requester = requester self.tester = None self.redirectRegExp = None self.invalidStatus = None self.dynamicParser = None self.ratio = 0.98 self.redirectStatusCodes = [301, 302, 307] self.setup()
def __init__(self, requester, testPath=None, suffix=None, preffix=None): if not testPath: self.testPath = RandomUtils.randString() else: self.testPath = testPath self.suffix = suffix if suffix else "" self.preffix = preffix if preffix else "" self.requester = requester self.tester = None self.redirectRegExp = None self.invalidStatus = None self.dynamicParser = None self.ratio = 0.98 self.setup()
def generateRedirectRegExp(self, firstLoc, firstPath, secondLoc, secondPath): # Use a unique sign to locate where the path gets reflected in the redirect self.sign = RandomUtils.randString(n=20) firstLoc = firstLoc.replace(firstPath, self.sign) secondLoc = secondLoc.replace(secondPath, self.sign) regExpStart = "^" regExpEnd = "$" for f, s in zip(firstLoc, secondLoc): if f == s: regExpStart += re.escape(f) else: regExpStart += ".*" break if regExpStart.endswith(".*"): for f, s in zip(firstLoc[::-1], secondLoc[::-1]): if f == s: regExpEnd = re.escape(f) + regExpEnd else: break return unquote(regExpStart + regExpEnd)
def __init__(self, script_path, arguments, output): global VERSION program_banner = open( FileUtils.buildPath(script_path, "lib", "controller", "banner.txt")).read().format(**VERSION) self.script_path = script_path self.exit = False self.arguments = arguments self.output = output self.savePath = self.script_path self.doneDirs = [] self.recursive_level_max = self.arguments.recursive_level_max if self.arguments.httpmethod.lower() not in ["get", "head", "post"]: self.output.error("Inavlid http method!") exit(1) self.httpmethod = self.arguments.httpmethod.lower() if self.arguments.saveHome: savePath = self.getSavePath() if not FileUtils.exists(savePath): FileUtils.createDirectory(savePath) if FileUtils.exists(savePath) and not FileUtils.isDir(savePath): self.output.error( 'Cannot use {} because is a file. Should be a directory'. format(savePath)) exit(1) if not FileUtils.canWrite(savePath): self.output.error( 'Directory {} is not writable'.format(savePath)) exit(1) logs = FileUtils.buildPath(savePath, "logs") if not FileUtils.exists(logs): FileUtils.createDirectory(logs) reports = FileUtils.buildPath(savePath, "reports") if not FileUtils.exists(reports): FileUtils.createDirectory(reports) self.savePath = savePath self.reportsPath = FileUtils.buildPath(self.savePath, "logs") self.blacklists = self.getBlacklists() self.blacklists = {} self.fuzzer = None self.excludeStatusCodes = self.arguments.excludeStatusCodes self.excludeTexts = self.arguments.excludeTexts self.excludeRegexps = self.arguments.excludeRegexps self.recursive = self.arguments.recursive self.suppressEmpty = self.arguments.suppressEmpty self.directories = Queue() self.excludeSubdirs = (arguments.excludeSubdirs if arguments.excludeSubdirs is not None else []) self.output.header(program_banner) # self.dictionary = Dictionary(self.arguments.wordlist, self.arguments.extensions, # self.arguments.lowercase, self.arguments.forceExtensions) # self.printConfig() self.errorLog = None self.errorLogPath = None self.errorLogLock = Lock() self.batch = False self.batchSession = None self.setupErrorLogs() self.output.newLine("\nError Log: {0}".format(self.errorLogPath)) if self.arguments.autoSave and len(self.arguments.urlList) > 1: self.setupBatchReports() self.output.newLine("\nAutoSave path: {0}".format( self.batchDirectoryPath)) if self.arguments.useRandomAgents: self.randomAgents = FileUtils.getLines( FileUtils.buildPath(script_path, "db", "user-agents.txt")) try: for url in self.arguments.urlList: try: gc.collect() self.reportManager = ReportManager() self.currentUrl = url self.output.target(self.currentUrl) try: # DNS A Record query self.requester = Requester( url, script_path=self.script_path, cookie=self.arguments.cookie, useragent=self.arguments.useragent, maxPool=self.arguments.threadsCount, maxRetries=self.arguments.maxRetries, delay=self.arguments.delay, timeout=self.arguments.timeout, ip=self.arguments.ip, proxy=self.arguments.proxy, redirect=self.arguments.redirect, requestByHostname=self.arguments.requestByHostname, httpmethod=self.httpmethod) # 网站连通性测试 site_connection_test_resp = self.requester.request( self.requester.basePath, use_base_path=False, allow_redirect=True, fingerprint=True) self.dictionary = Dictionary(self.requester.scan_list, self.requester.directory, self.requester.filename, self.requester.extension) # 404 page if self.requester.url_type == URLType.normal_restful_dir: path_404 = '{}/{}/'.format( self.requester.basePath, RandomUtils.randString(8)) path_404 = path_404.replace("//", "/") elif self.requester.url_type == URLType.restful_file: path_404 = self.requester.basePath.replace( self.requester.filename, RandomUtils.randString( len(self.requester.filename) or 8)) elif self.requester.url_type == URLType.normal_file: path_404 = self.requester.basePath.replace( self.requester.filename, RandomUtils.randString( len(self.requester.filename) or 8)) path_404_quote = self.dictionary.quote(path_404) response_404 = self.requester.request( path_404_quote, use_base_path=False, allow_redirect=False) # Waf 探测 waf_exist, waf_response = self.requester.waf_detect( site_connection_test_resp.body, url_quote=self.dictionary.quote) except RequestException as e: self.output.error(e.args[0]['message']) raise SkipTargetInterrupt if self.arguments.useRandomAgents: self.requester.setRandomAgents(self.randomAgents) for key, value in arguments.headers.items(): self.requester.setHeader(key, value) # Initialize directories Queue with start Path self.basePath = self.requester.basePath if self.arguments.scanSubdirs is not None: for subdir in self.arguments.scanSubdirs: self.directories.put(subdir) else: self.directories.put('') self.setupReports(self.requester) matchCallbacks = [self.matchCallback] notFoundCallbacks = [self.notFoundCallback] errorCallbacks = [self.errorCallback, self.appendErrorLog] self.fuzzer = Fuzzer( self.requester, self.dictionary, waf_exist, waf_response, response_404, testFailPath=self.arguments.testFailPath, threads=self.arguments.threadsCount, matchCallbacks=matchCallbacks, notFoundCallbacks=notFoundCallbacks, errorCallbacks=errorCallbacks) try: self.wait() except RequestException as e: self.output.error( "Fatal error during site scanning: " + e.args[0]['message']) raise SkipTargetInterrupt except SkipTargetInterrupt: continue finally: self.reportManager.save() except KeyboardInterrupt: self.output.error('\nCanceled by the user') exit(0) finally: if not self.errorLog.closed: self.errorLog.close() self.reportManager.close() self.output.warning('\nTask Completed')