예제 #1
0
    def setup(self):
        first_path = self.prefix + (self.calibration if self.calibration else
                                    RandomUtils.rand_string()) + self.suffix
        first_response = self.requester.request(first_path)
        self.invalid_status = first_response.status

        if self.invalid_status == 404:
            # Using the response status code is enough :-}
            return

        second_path = self.prefix + (self.calibration if self.calibration else
                                     RandomUtils.rand_string(
                                         omit=first_path)) + self.suffix
        second_response = self.requester.request(second_path)

        # Look for redirects
        if first_response.redirect and second_response.redirect:
            self.redirect_reg_exp = self.generate_redirect_reg_exp(
                first_response.redirect,
                first_path,
                second_response.redirect,
                second_path,
            )

        # Analyze response bodies
        if first_response.body is not None and second_response.body is not None:
            self.dynamic_parser = DynamicContentParser(self.requester,
                                                       first_path,
                                                       first_response.body,
                                                       second_response.body)
        else:
            self.dynamic_parser = None

        self.ratio = float("{0:.2f}".format(
            self.dynamic_parser.comparisonRatio))  # Rounding to 2 decimals

        # The wildcard response is static
        if self.ratio == 1:
            pass
        # Adjusting ratio based on response length
        elif len(first_response) < 100:
            self.ratio -= 0.1
        elif len(first_response) < 500:
            self.ratio -= 0.05
        elif len(first_response) < 2000:
            self.ratio -= 0.02
        else:
            self.ratio -= 0.01
        # If the path is reflected in response, decrease the ratio. Because
        # the difference between path lengths can reduce the similarity ratio
        if first_path in first_response.body.decode(
        ) and len(first_response) < 100000:
            self.ratio -= 0.1
예제 #2
0
    def setup(self):
        firstPath = self.preffix + self.testPath + self.suffix
        firstResponse = self.requester.request(firstPath)
        self.invalidStatus = firstResponse.status

        secondPath = self.preffix + RandomUtils.randString(
            omit=self.testPath) + self.suffix
        secondResponse = self.requester.request(secondPath)

        if self.invalidStatus == 404:
            # Using the response status code is enough :-}
            return

        # look for redirects
        elif firstResponse.status in self.redirectStatusCodes and firstResponse.redirect and secondResponse.redirect:
            self.redirectRegExp = self.generateRedirectRegExp(
                firstResponse.redirect, secondResponse.redirect)

        # Analyze response bodies
        self.dynamicParser = DynamicContentParser(self.requester, firstPath,
                                                  firstResponse.body,
                                                  secondResponse.body)

        baseRatio = float("{0:.2f}".format(
            self.dynamicParser.comparisonRatio))  # Rounding to 2 decimals

        # If response length is small, adjust ratio
        if len(firstResponse) < 2000:
            baseRatio -= 0.1

        if baseRatio < self.ratio:
            self.ratio = baseRatio
예제 #3
0
    def setup(self):
        firstPath = self.testPath + self.suffix
        firstResponse = self.requester.request(firstPath)
        self.invalidStatus = firstResponse.status

        if self.invalidStatus == 404:
            # Using the response status code is enough :-}
            return

        # look for redirects
        secondPath = RandomUtils.randString(omit=self.testPath) + self.suffix
        secondResponse = self.requester.request(secondPath)

        if firstResponse.status in self.redirectStatusCodes and firstResponse.redirect and secondResponse.redirect:
            self.redirectRegExp = self.generateRedirectRegExp(firstResponse.redirect, secondResponse.redirect)

        # Analyze response bodies
        self.dynamicParser = DynamicContentParser(self.requester, firstPath, firstResponse.body, secondResponse.body)

        baseRatio = float("{0:.2f}".format(self.dynamicParser.comparisonRatio))  # Rounding to 2 decimals

        # If response length is small, adjust ratio
        if len(firstResponse) < 2000:
            baseRatio -= 0.1

        if baseRatio < self.ratio:
            self.ratio = baseRatio
예제 #4
0
    def setup(self):
        for path in self.testPath:
            firstPath = RandomUtils.randString() + '/' + path + self.suffix
            firstResponse = self.requester.request(firstPath)
            if firstResponse.status not in self.invalidStatus:
                self.invalidStatus.append(firstResponse.status)

            if firstResponse.status == 404:
                # Using the response status code is enough :-}
                continue

            # look for redirects
            secondPath = RandomUtils.randString() + '/' + path + self.suffix
            secondResponse = self.requester.request(secondPath)

            if firstResponse.status in self.redirectStatusCodes and firstResponse.redirect and secondResponse.redirect:
                self.redirectRegExp.append(
                    self.generateRedirectRegExp(firstResponse.redirect,
                                                secondResponse.redirect))

            # Analyze response bodies
            dynamicParser = DynamicContentParser(self.requester, firstPath,
                                                 firstResponse.body,
                                                 secondResponse.body)

            baseRatio = float("{0:.2f}".format(
                dynamicParser.comparisonRatio))  # Rounding to 2 decimals

            ratio = self.ratio
            # If response length is small, adjust ratio
            if len(firstResponse) < 2000:
                baseRatio -= 0.1

            if baseRatio < self.ratio:
                ratio = baseRatio

            if self.dynamicParser:
                flag = 0
                for _dynamicParser, __ in self.dynamicParser:
                    _ratio = dynamicParser.compareTo(_dynamicParser.cleanPage)
                    flag += _ratio > ratio

                if not flag:
                    self.dynamicParser.append((dynamicParser, ratio))

            else:
                self.dynamicParser.append((dynamicParser, ratio))
예제 #5
0
    def get_special_str(self, str):
        str_list = list(str)
        str_set_len = len(set(str_list))

        for _ in range(100):
            if str_set_len == 1 and len(str_list) == 1:
                new_str = RandomUtils.randString(n=1, omit=str_list)
            elif str_set_len == 1:
                _ = RandomUtils.randString(n=len(str_list) - str_set_len)
                new_str = '{}{}'.format(str_list[0], _)
            else:
                random.shuffle(str_list)
                new_str = ''.join(str_list)
            if new_str == str:
                continue
            return new_str
        else:
            raise Exception('Special Str: {}'.format(str))
예제 #6
0
    def setup(self):
        first_path = self.prefix + (
            self.calibration if self.calibration else RandomUtils.rand_string()
        ) + self.suffix
        first_response = self.requester.request(first_path)
        self.invalid_status = first_response.status

        if self.invalid_status == 404:
            # Using the response status code is enough :-}
            return

        second_path = self.prefix + (
            self.calibration if self.calibration else RandomUtils.rand_string(omit=first_path)
        ) + self.suffix
        second_response = self.requester.request(second_path)

        # Look for redirects
        if first_response.redirect and second_response.redirect:
            self.redirect_reg_exp = self.generate_redirect_reg_exp(
                first_response.redirect, first_path,
                second_response.redirect, second_path,
            )

        # Analyze response bodies
        if first_response.body is not None and second_response.body is not None:
            self.dynamic_parser = DynamicContentParser(
                self.requester, first_path, first_response.body, second_response.body
            )
        else:
            self.dynamic_parser = None

        base_ratio = float(
            "{0:.2f}".format(self.dynamic_parser.comparisonRatio)
        )  # Rounding to 2 decimals

        # If response length is small, adjust ratio
        if len(first_response) < 500:
            base_ratio -= 0.15
        elif len(first_response) < 2000:
            base_ratio -= 0.1

        if base_ratio < self.ratio:
            self.ratio = base_ratio
예제 #7
0
    def get_special_path(self, path):
        special_path = []
        if path.endswith('/'):
            _origin_path, _dir = path[:-1].rsplit('/', 1)
        else:
            _origin_path, _last_element = path.rsplit('/', 1)

        if path.endswith('/'):
            # unzip exception
            _origin_path, _dir = path[:-1].rsplit('/', 1)
            special_dir = self.get_special_str(_dir)
            _path = '{}/{}/'.format(_origin_path, special_dir)
            if not _path.startswith('/'):
                _path = '/' + _path
            special_path.append(_path)
        elif '.' in _last_element:
            _origin_path, _filename_ext = path.rsplit('/', 1)
            _filename, _ext = _filename_ext.rsplit('.', 1)
            # special filename
            if not _filename:
                _filename = RandomUtils.randString(n=8)
            special_filename = self.get_special_str(_filename)
            _path = '{}/{}.{}'.format(_origin_path, special_filename, _ext)
            if not _path.startswith('/'):
                _path = '/' + _path
            special_path.append(_path)
            # special extension
            if not _ext:
                _ext = RandomUtils.randString(n=8)
            special_extension = self.get_special_str(_ext)
            _path = '{}/{}.{}'.format(_origin_path, _filename,
                                      special_extension)
            if not _path.startswith('/'):
                _path = '/' + _path
            special_path.append(_path)
        else:
            _origin_path, _filename = path.rsplit('/', 1)
            special_filename = self.get_special_str(_filename)
            _path = '{}/{}'.format(_origin_path, special_filename)
            if not _path.startswith('/'):
                _path = '/' + _path
            special_path.append(_path)
        return special_path
예제 #8
0
 def __init__(self, requester, testPath=None, suffix=None):
     if testPath is None or testPath is "":
         self.testPath = RandomUtils.randString()
     else:
         self.testPath = testPath
     self.suffix = suffix if suffix is not None else ""
     self.requester = requester
     self.tester = None
     self.redirectRegExp = None
     self.invalidStatus = None
     self.dynamicParser = None
     self.ratio = 0.98
     self.redirectStatusCodes = [301, 302, 307]
     self.setup()
예제 #9
0
 def __init__(self, requester, testPath=None, suffix=None):
     if testPath is None or testPath is "":
         self.testPath = RandomUtils.randString()
     else:
         self.testPath = testPath
     self.suffix = suffix if suffix is not None else ""
     self.requester = requester
     self.tester = None
     self.redirectRegExp = None
     self.invalidStatus = None
     self.dynamicParser = None
     self.ratio = 0.98
     self.redirectStatusCodes = [301, 302, 307]
     self.setup()
예제 #10
0
    def __init__(self, requester, testPath=None, suffix=None, preffix=None):
        if not testPath:
            self.testPath = RandomUtils.randString()
        else:
            self.testPath = testPath

        self.suffix = suffix if suffix else ""
        self.preffix = preffix if preffix else ""
        self.requester = requester
        self.tester = None
        self.redirectRegExp = None
        self.invalidStatus = None
        self.dynamicParser = None
        self.ratio = 0.98
        self.setup()
예제 #11
0
파일: scanner.py 프로젝트: jsfan/dirsearch
    def generate_redirect_reg_exp(self, first_loc, first_path, second_loc, second_path):
        # Use a unique sign to locate where the path gets reflected in the redirect
        self.sign = RandomUtils.rand_string(n=20)
        first_loc = first_loc.replace(first_path, self.sign)
        second_loc = second_loc.replace(second_path, self.sign)
        reg_exp_start = "^"
        reg_exp_end = "$"

        for f, s in zip(first_loc, second_loc):
            if f == s:
                reg_exp_start += re.escape(f)
            else:
                reg_exp_start += ".*"
                break

        if reg_exp_start.endswith(".*"):
            for f, s in zip(first_loc[::-1], second_loc[::-1]):
                if f == s:
                    reg_exp_end = re.escape(f) + reg_exp_end
                else:
                    break

        return unquote(reg_exp_start + reg_exp_end)
예제 #12
0
    def __init__(self, script_path, arguments, output):
        global VERSION
        program_banner = open(
            FileUtils.buildPath(script_path, "lib", "controller",
                                "banner.txt")).read().format(**VERSION)

        self.script_path = script_path
        self.exit = False
        self.arguments = arguments
        self.output = output
        self.savePath = self.script_path
        self.doneDirs = []

        self.recursive_level_max = self.arguments.recursive_level_max

        if self.arguments.httpmethod.lower() not in ["get", "head", "post"]:
            self.output.error("Inavlid http method!")
            exit(1)

        self.httpmethod = self.arguments.httpmethod.lower()

        if self.arguments.saveHome:
            savePath = self.getSavePath()

            if not FileUtils.exists(savePath):
                FileUtils.createDirectory(savePath)

            if FileUtils.exists(savePath) and not FileUtils.isDir(savePath):
                self.output.error(
                    'Cannot use {} because is a file. Should be a directory'.
                    format(savePath))
                exit(1)

            if not FileUtils.canWrite(savePath):
                self.output.error(
                    'Directory {} is not writable'.format(savePath))
                exit(1)

            logs = FileUtils.buildPath(savePath, "logs")

            if not FileUtils.exists(logs):
                FileUtils.createDirectory(logs)

            reports = FileUtils.buildPath(savePath, "reports")

            if not FileUtils.exists(reports):
                FileUtils.createDirectory(reports)

            self.savePath = savePath

        self.reportsPath = FileUtils.buildPath(self.savePath, "logs")
        self.blacklists = self.getBlacklists()
        self.blacklists = {}
        self.fuzzer = None
        self.excludeStatusCodes = self.arguments.excludeStatusCodes
        self.excludeTexts = self.arguments.excludeTexts
        self.excludeRegexps = self.arguments.excludeRegexps
        self.recursive = self.arguments.recursive
        self.suppressEmpty = self.arguments.suppressEmpty
        self.directories = Queue()
        self.excludeSubdirs = (arguments.excludeSubdirs
                               if arguments.excludeSubdirs is not None else [])
        self.output.header(program_banner)
        # self.dictionary = Dictionary(self.arguments.wordlist, self.arguments.extensions,
        #                              self.arguments.lowercase, self.arguments.forceExtensions)
        # self.printConfig()
        self.errorLog = None
        self.errorLogPath = None
        self.errorLogLock = Lock()
        self.batch = False
        self.batchSession = None
        self.setupErrorLogs()
        self.output.newLine("\nError Log: {0}".format(self.errorLogPath))

        if self.arguments.autoSave and len(self.arguments.urlList) > 1:
            self.setupBatchReports()
            self.output.newLine("\nAutoSave path: {0}".format(
                self.batchDirectoryPath))

        if self.arguments.useRandomAgents:
            self.randomAgents = FileUtils.getLines(
                FileUtils.buildPath(script_path, "db", "user-agents.txt"))

        try:
            for url in self.arguments.urlList:

                try:
                    gc.collect()
                    self.reportManager = ReportManager()
                    self.currentUrl = url
                    self.output.target(self.currentUrl)

                    try:
                        # DNS A Record query
                        self.requester = Requester(
                            url,
                            script_path=self.script_path,
                            cookie=self.arguments.cookie,
                            useragent=self.arguments.useragent,
                            maxPool=self.arguments.threadsCount,
                            maxRetries=self.arguments.maxRetries,
                            delay=self.arguments.delay,
                            timeout=self.arguments.timeout,
                            ip=self.arguments.ip,
                            proxy=self.arguments.proxy,
                            redirect=self.arguments.redirect,
                            requestByHostname=self.arguments.requestByHostname,
                            httpmethod=self.httpmethod)
                        # 网站连通性测试
                        site_connection_test_resp = self.requester.request(
                            self.requester.basePath,
                            use_base_path=False,
                            allow_redirect=True,
                            fingerprint=True)
                        self.dictionary = Dictionary(self.requester.scan_list,
                                                     self.requester.directory,
                                                     self.requester.filename,
                                                     self.requester.extension)
                        # 404 page
                        if self.requester.url_type == URLType.normal_restful_dir:
                            path_404 = '{}/{}/'.format(
                                self.requester.basePath,
                                RandomUtils.randString(8))
                            path_404 = path_404.replace("//", "/")
                        elif self.requester.url_type == URLType.restful_file:
                            path_404 = self.requester.basePath.replace(
                                self.requester.filename,
                                RandomUtils.randString(
                                    len(self.requester.filename) or 8))
                        elif self.requester.url_type == URLType.normal_file:
                            path_404 = self.requester.basePath.replace(
                                self.requester.filename,
                                RandomUtils.randString(
                                    len(self.requester.filename) or 8))
                        path_404_quote = self.dictionary.quote(path_404)
                        response_404 = self.requester.request(
                            path_404_quote,
                            use_base_path=False,
                            allow_redirect=False)

                        # Waf 探测
                        waf_exist, waf_response = self.requester.waf_detect(
                            site_connection_test_resp.body,
                            url_quote=self.dictionary.quote)

                    except RequestException as e:
                        self.output.error(e.args[0]['message'])
                        raise SkipTargetInterrupt

                    if self.arguments.useRandomAgents:
                        self.requester.setRandomAgents(self.randomAgents)

                    for key, value in arguments.headers.items():
                        self.requester.setHeader(key, value)

                    # Initialize directories Queue with start Path
                    self.basePath = self.requester.basePath

                    if self.arguments.scanSubdirs is not None:
                        for subdir in self.arguments.scanSubdirs:
                            self.directories.put(subdir)

                    else:
                        self.directories.put('')

                    self.setupReports(self.requester)

                    matchCallbacks = [self.matchCallback]
                    notFoundCallbacks = [self.notFoundCallback]
                    errorCallbacks = [self.errorCallback, self.appendErrorLog]

                    self.fuzzer = Fuzzer(
                        self.requester,
                        self.dictionary,
                        waf_exist,
                        waf_response,
                        response_404,
                        testFailPath=self.arguments.testFailPath,
                        threads=self.arguments.threadsCount,
                        matchCallbacks=matchCallbacks,
                        notFoundCallbacks=notFoundCallbacks,
                        errorCallbacks=errorCallbacks)
                    try:
                        self.wait()
                    except RequestException as e:
                        self.output.error(
                            "Fatal error during site scanning: " +
                            e.args[0]['message'])
                        raise SkipTargetInterrupt

                except SkipTargetInterrupt:
                    continue

                finally:
                    self.reportManager.save()

        except KeyboardInterrupt:
            self.output.error('\nCanceled by the user')
            exit(0)

        finally:
            if not self.errorLog.closed:
                self.errorLog.close()

            self.reportManager.close()

        self.output.warning('\nTask Completed')