Exemple #1
0
    def setup(self):
        firstPath = self.preffix + self.testPath + self.suffix
        firstResponse = self.requester.request(firstPath)
        self.invalidStatus = firstResponse.status

        secondPath = self.preffix + RandomUtils.randString(
            omit=self.testPath) + self.suffix
        secondResponse = self.requester.request(secondPath)

        if self.invalidStatus == 404:
            # Using the response status code is enough :-}
            return

        # look for redirects
        elif firstResponse.status in self.redirectStatusCodes and firstResponse.redirect and secondResponse.redirect:
            self.redirectRegExp = self.generateRedirectRegExp(
                firstResponse.redirect, secondResponse.redirect)

        # Analyze response bodies
        self.dynamicParser = DynamicContentParser(self.requester, firstPath,
                                                  firstResponse.body,
                                                  secondResponse.body)

        baseRatio = float("{0:.2f}".format(
            self.dynamicParser.comparisonRatio))  # Rounding to 2 decimals

        # If response length is small, adjust ratio
        if len(firstResponse) < 2000:
            baseRatio -= 0.1

        if baseRatio < self.ratio:
            self.ratio = baseRatio
Exemple #2
0
    def setup(self):
        first_path = self.prefix + (self.calibration if self.calibration else
                                    RandomUtils.rand_string()) + self.suffix
        first_response = self.requester.request(first_path)
        self.invalid_status = first_response.status

        if self.invalid_status == 404:
            # Using the response status code is enough :-}
            return

        second_path = self.prefix + (self.calibration if self.calibration else
                                     RandomUtils.rand_string(
                                         omit=first_path)) + self.suffix
        second_response = self.requester.request(second_path)

        # Look for redirects
        if first_response.redirect and second_response.redirect:
            self.redirect_reg_exp = self.generate_redirect_reg_exp(
                first_response.redirect,
                first_path,
                second_response.redirect,
                second_path,
            )

        # Analyze response bodies
        if first_response.body is not None and second_response.body is not None:
            self.dynamic_parser = DynamicContentParser(self.requester,
                                                       first_path,
                                                       first_response.body,
                                                       second_response.body)
        else:
            self.dynamic_parser = None

        self.ratio = float("{0:.2f}".format(
            self.dynamic_parser.comparisonRatio))  # Rounding to 2 decimals

        # The wildcard response is static
        if self.ratio == 1:
            pass
        # Adjusting ratio based on response length
        elif len(first_response) < 100:
            self.ratio -= 0.1
        elif len(first_response) < 500:
            self.ratio -= 0.05
        elif len(first_response) < 2000:
            self.ratio -= 0.02
        else:
            self.ratio -= 0.01
        # If the path is reflected in response, decrease the ratio. Because
        # the difference between path lengths can reduce the similarity ratio
        if first_path in first_response.body.decode(
        ) and len(first_response) < 100000:
            self.ratio -= 0.1
Exemple #3
0
    def setup(self):
        firstPath = self.testPath + self.suffix
        firstResponse = self.requester.request(firstPath)
        self.invalidStatus = firstResponse.status

        if self.invalidStatus == 404:
            # Using the response status code is enough :-}
            return

        # look for redirects
        secondPath = RandomUtils.randString(omit=self.testPath) + self.suffix
        secondResponse = self.requester.request(secondPath)

        if firstResponse.status in self.redirectStatusCodes and firstResponse.redirect and secondResponse.redirect:
            self.redirectRegExp = self.generateRedirectRegExp(firstResponse.redirect, secondResponse.redirect)

        # Analyze response bodies
        self.dynamicParser = DynamicContentParser(self.requester, firstPath, firstResponse.body, secondResponse.body)

        baseRatio = float("{0:.2f}".format(self.dynamicParser.comparisonRatio))  # Rounding to 2 decimals

        # If response length is small, adjust ratio
        if len(firstResponse) < 2000:
            baseRatio -= 0.1

        if baseRatio < self.ratio:
            self.ratio = baseRatio
Exemple #4
0
    def setup(self):
        for path in self.testPath:
            firstPath = RandomUtils.randString() + '/' + path + self.suffix
            firstResponse = self.requester.request(firstPath)
            if firstResponse.status not in self.invalidStatus:
                self.invalidStatus.append(firstResponse.status)

            if firstResponse.status == 404:
                # Using the response status code is enough :-}
                continue

            # look for redirects
            secondPath = RandomUtils.randString() + '/' + path + self.suffix
            secondResponse = self.requester.request(secondPath)

            if firstResponse.status in self.redirectStatusCodes and firstResponse.redirect and secondResponse.redirect:
                self.redirectRegExp.append(
                    self.generateRedirectRegExp(firstResponse.redirect,
                                                secondResponse.redirect))

            # Analyze response bodies
            dynamicParser = DynamicContentParser(self.requester, firstPath,
                                                 firstResponse.body,
                                                 secondResponse.body)

            baseRatio = float("{0:.2f}".format(
                dynamicParser.comparisonRatio))  # Rounding to 2 decimals

            ratio = self.ratio
            # If response length is small, adjust ratio
            if len(firstResponse) < 2000:
                baseRatio -= 0.1

            if baseRatio < self.ratio:
                ratio = baseRatio

            if self.dynamicParser:
                flag = 0
                for _dynamicParser, __ in self.dynamicParser:
                    _ratio = dynamicParser.compareTo(_dynamicParser.cleanPage)
                    flag += _ratio > ratio

                if not flag:
                    self.dynamicParser.append((dynamicParser, ratio))

            else:
                self.dynamicParser.append((dynamicParser, ratio))
    def setup(self):
        first_path = self.prefix + (
            self.calibration if self.calibration else RandomUtils.rand_string()
        ) + self.suffix
        first_response = self.requester.request(first_path)
        self.invalid_status = first_response.status

        if self.invalid_status == 404:
            # Using the response status code is enough :-}
            return

        second_path = self.prefix + (
            self.calibration if self.calibration else RandomUtils.rand_string(omit=first_path)
        ) + self.suffix
        second_response = self.requester.request(second_path)

        # Look for redirects
        if first_response.redirect and second_response.redirect:
            self.redirect_reg_exp = self.generate_redirect_reg_exp(
                first_response.redirect, first_path,
                second_response.redirect, second_path,
            )

        # Analyze response bodies
        if first_response.body is not None and second_response.body is not None:
            self.dynamic_parser = DynamicContentParser(
                self.requester, first_path, first_response.body, second_response.body
            )
        else:
            self.dynamic_parser = None

        base_ratio = float(
            "{0:.2f}".format(self.dynamic_parser.comparisonRatio)
        )  # Rounding to 2 decimals

        # If response length is small, adjust ratio
        if len(first_response) < 500:
            base_ratio -= 0.15
        elif len(first_response) < 2000:
            base_ratio -= 0.1

        if base_ratio < self.ratio:
            self.ratio = base_ratio
Exemple #6
0
class Scanner(object):
    def __init__(self, requester, testPath=None, suffix=None, preffix=None):
        if testPath is None or testPath == "":
            self.testPath = RandomUtils.randString()
        else:
            self.testPath = testPath

        self.suffix = suffix if suffix else ""
        self.preffix = preffix if preffix else ""
        self.requester = requester
        self.tester = None
        self.redirectRegExp = None
        self.invalidStatus = None
        self.dynamicParser = None
        self.ratio = 0.98
        self.redirectStatusCodes = [301, 302, 303, 307, 308]
        self.setup()

    def setup(self):
        firstPath = self.preffix + self.testPath + self.suffix
        firstResponse = self.requester.request(firstPath)
        self.invalidStatus = firstResponse.status

        secondPath = self.preffix + RandomUtils.randString(
            omit=self.testPath) + self.suffix
        secondResponse = self.requester.request(secondPath)

        if self.invalidStatus == 404:
            # Using the response status code is enough :-}
            return

        # look for redirects
        elif firstResponse.status in self.redirectStatusCodes and firstResponse.redirect and secondResponse.redirect:
            self.redirectRegExp = self.generateRedirectRegExp(
                firstResponse.redirect, secondResponse.redirect)

        # Analyze response bodies
        self.dynamicParser = DynamicContentParser(self.requester, firstPath,
                                                  firstResponse.body,
                                                  secondResponse.body)

        baseRatio = float("{0:.2f}".format(
            self.dynamicParser.comparisonRatio))  # Rounding to 2 decimals

        # If response length is small, adjust ratio
        if len(firstResponse) < 2000:
            baseRatio -= 0.1

        if baseRatio < self.ratio:
            self.ratio = baseRatio

    def generateRedirectRegExp(self, firstLocation, secondLocation):
        if firstLocation is None or secondLocation is None:
            return None

        sm = SequenceMatcher(None, firstLocation, secondLocation)
        marks = []

        for blocks in sm.get_matching_blocks():
            i = blocks[0]
            n = blocks[2]
            # empty block

            if n == 0:
                continue

            mark = firstLocation[i:i + n]
            marks.append(mark)

        regexp = "^.*{0}.*$".format(".*".join(map(re.escape, marks)))
        return regexp

    def scan(self, path, response):
        if self.invalidStatus == response.status == 404:
            return False

        if self.invalidStatus != response.status:
            return True

        redirectToInvalid = False

        if self.redirectRegExp and response.redirect:
            redirectToInvalid = (re.match(self.redirectRegExp,
                                          response.redirect) is not None)
            # If redirection doesn't match the rule, mark as found

            if not redirectToInvalid:
                return True

        ratio = self.dynamicParser.compareTo(response.body)

        if ratio >= self.ratio:
            return False

        elif redirectToInvalid and ratio >= (self.ratio - 0.15):
            return False

        return True
Exemple #7
0
class Scanner(object):
    def __init__(self, requester, calibration=None, suffix=None, prefix=None):
        self.calibration = calibration
        self.suffix = suffix if suffix else ""
        self.prefix = prefix if prefix else ""
        self.requester = requester
        self.tester = None
        self.redirectRegExp = None
        self.invalidStatus = None
        self.dynamicParser = None
        self.sign = None
        self.ratio = 0.98
        self.setup()

    def setup(self):
        firstPath = self.prefix + (self.calibration if self.calibration else
                                   RandomUtils.randString()) + self.suffix
        firstResponse = self.requester.request(firstPath)
        self.invalidStatus = firstResponse.status

        if self.invalidStatus == 404:
            # Using the response status code is enough :-}
            return

        secondPath = self.prefix + (self.calibration if self.calibration else
                                    RandomUtils.randString(
                                        omit=firstPath)) + self.suffix
        secondResponse = self.requester.request(secondPath)

        # Look for redirects
        if firstResponse.redirect and secondResponse.redirect:
            self.redirectRegExp = self.generateRedirectRegExp(
                firstResponse.redirect, secondResponse.redirect)

        # Analyze response bodies
        if firstResponse.body is not None and secondResponse.body is not None:
            self.dynamicParser = DynamicContentParser(self.requester,
                                                      firstPath,
                                                      firstResponse.body,
                                                      secondResponse.body)
        else:
            self.dynamicParser = None

        baseRatio = float("{0:.2f}".format(
            self.dynamicParser.comparisonRatio))  # Rounding to 2 decimals

        # If response length is small, adjust ratio
        if len(firstResponse) < 2000:
            baseRatio -= 0.1

        if baseRatio < self.ratio:
            self.ratio = baseRatio

    def generateRedirectRegExp(self, firstLoc, firstPath, secondLoc,
                               secondPath):
        # Use a unique sign to locate where the path gets reflected in the redirect
        self.sign = RandomUtils.randString(n=20)
        firstLoc = firstLoc.replace(firstPath, self.sign)
        secondLoc = secondLoc.replace(secondPath, self.sign)
        regExpStart = "^"
        regExpEnd = "$"

        for f, s in zip(firstLoc, secondLoc):
            if f == s:
                regExpStart += re.escape(f)
            else:
                regExpStart += ".*"
                break

        if regExpStart.endswith(".*"):
            for f, s in zip(firstLoc[::-1], secondLoc[::-1]):
                if f == s:
                    regExpEnd = re.escape(f) + regExpEnd
                else:
                    break

        return unquote(regExpStart + regExpEnd)

    def scan(self, path, response):
        if self.invalidStatus == response.status == 404:
            return False

        if self.invalidStatus != response.status:
            return True

        if self.redirectRegExp and response.redirect:
            path = re.escape(unquote(path))
            # A lot of times, '#' or '?' will be removed in the redirect, cause false positives
            for char in ["\\#", "\\?"]:
                if char in path:
                    path = path.replace(char, "(|" + char) + ")"

            redirectRegExp = self.redirectRegExp.replace(self.sign, path)

            # Redirect sometimes encodes/decodes characters in URL, which may confuse the
            # rule check and make noise in the output, so we need to unquote() everything
            redirectToInvalid = re.match(redirectRegExp,
                                         unquote(response.redirect))

            # If redirection doesn't match the rule, mark as found
            if redirectToInvalid is None:
                return True

        ratio = self.dynamicParser.compareTo(response.body)

        if ratio >= self.ratio:
            return False

        elif "redirectToInvalid" in locals() and ratio >= (self.ratio - 0.15):
            return False

        return True
Exemple #8
0
    def setup(self):
        first_path = self.prefix + (self.calibration if self.calibration else
                                    rand_string()) + self.suffix
        first_response = self.requester.request(first_path)
        self.response = first_response

        if self.response.status == 404:
            # Using the response status code is enough :-}
            return

        duplicate = self.duplicate(first_response)
        if duplicate:
            # Another test had been performed and shows the same response as this
            self.ratio = duplicate.ratio
            self.dynamic_parser = duplicate.dynamic_parser
            self.redirect_parser = duplicate.redirect_parser
            self.sign = duplicate.sign
            return

        second_path = self.prefix + (self.calibration
                                     if self.calibration else rand_string(
                                         omit=first_path)) + self.suffix
        second_response = self.requester.request(second_path)

        if first_response.redirect and second_response.redirect:
            self.generate_redirect_reg_exp(
                first_response.redirect,
                first_path,
                second_response.redirect,
                second_path,
            )

        # Analyze response bodies
        if first_response.body is not None and second_response.body is not None:
            self.dynamic_parser = DynamicContentParser(self.requester,
                                                       first_path,
                                                       first_response.body,
                                                       second_response.body)
        else:
            self.dynamic_parser = None

        self.ratio = float("{0:.2f}".format(
            self.dynamic_parser.comparisonRatio))  # Rounding to 2 decimals

        # The wildcard response is static
        if self.ratio == 1:
            pass
        # Adjusting ratio based on response length
        elif len(first_response) < 100:
            self.ratio -= 0.1
        elif len(first_response) < 500:
            self.ratio -= 0.05
        elif len(first_response) < 2000:
            self.ratio -= 0.02
        else:
            self.ratio -= 0.01
        """
        If the path is reflected in response, decrease the ratio. Because
        the difference between path lengths can reduce the similarity ratio
        """
        if first_path in first_response.body.decode():
            if len(first_response) < 200:
                self.ratio -= 0.15 + 15 / len(first_response)
            elif len(first_response) < 800:
                self.ratio -= 0.06 + 30 / len(first_response)
            elif len(first_response) < 5000:
                self.ratio -= 0.03 + 80 / len(first_response)
            elif len(first_response) < 20000:
                self.ratio -= 0.02 + 200 / len(first_response)
            else:
                self.ratio -= 0.01
Exemple #9
0
class Scanner(object):
    def __init__(self,
                 requester,
                 calibration=None,
                 suffix=None,
                 prefix=None,
                 tested=None):
        self.calibration = calibration
        self.suffix = suffix if suffix else ""
        self.prefix = prefix if prefix else ""
        self.tested = tested
        self.requester = requester
        self.tester = None
        self.response = None
        self.dynamic_parser = None
        self.redirect_parser = None
        self.sign = None
        self.setup()

    def duplicate(self, response):
        if not self.tested:
            return
        for t in self.tested:
            for tester in self.tested[t].values():
                if [response.status, response.body, response.redirect] == [
                        tester.response.status, tester.response.body,
                        tester.response.redirect
                ]:
                    return tester

        return

    """
    Generate wildcard response information containers, this will be
    used to compare with other path responses
    """

    def setup(self):
        first_path = self.prefix + (self.calibration if self.calibration else
                                    rand_string()) + self.suffix
        first_response = self.requester.request(first_path)
        self.response = first_response

        if self.response.status == 404:
            # Using the response status code is enough :-}
            return

        duplicate = self.duplicate(first_response)
        if duplicate:
            # Another test had been performed and shows the same response as this
            self.ratio = duplicate.ratio
            self.dynamic_parser = duplicate.dynamic_parser
            self.redirect_parser = duplicate.redirect_parser
            self.sign = duplicate.sign
            return

        second_path = self.prefix + (self.calibration
                                     if self.calibration else rand_string(
                                         omit=first_path)) + self.suffix
        second_response = self.requester.request(second_path)

        if first_response.redirect and second_response.redirect:
            self.generate_redirect_reg_exp(
                first_response.redirect,
                first_path,
                second_response.redirect,
                second_path,
            )

        # Analyze response bodies
        if first_response.body is not None and second_response.body is not None:
            self.dynamic_parser = DynamicContentParser(self.requester,
                                                       first_path,
                                                       first_response.body,
                                                       second_response.body)
        else:
            self.dynamic_parser = None

        self.ratio = float("{0:.2f}".format(
            self.dynamic_parser.comparisonRatio))  # Rounding to 2 decimals

        # The wildcard response is static
        if self.ratio == 1:
            pass
        # Adjusting ratio based on response length
        elif len(first_response) < 100:
            self.ratio -= 0.1
        elif len(first_response) < 500:
            self.ratio -= 0.05
        elif len(first_response) < 2000:
            self.ratio -= 0.02
        else:
            self.ratio -= 0.01
        """
        If the path is reflected in response, decrease the ratio. Because
        the difference between path lengths can reduce the similarity ratio
        """
        if first_path in first_response.body.decode():
            if len(first_response) < 200:
                self.ratio -= 0.15 + 15 / len(first_response)
            elif len(first_response) < 800:
                self.ratio -= 0.06 + 30 / len(first_response)
            elif len(first_response) < 5000:
                self.ratio -= 0.03 + 80 / len(first_response)
            elif len(first_response) < 20000:
                self.ratio -= 0.02 + 200 / len(first_response)
            else:
                self.ratio -= 0.01

    """
    From 2 redirects of wildcard responses, generate a regexp that matches
    every wildcard redirect
    """

    def generate_redirect_reg_exp(self, first_loc, first_path, second_loc,
                                  second_path):
        # Use a unique sign to locate where the path gets reflected in the redirect
        self.sign = rand_string(n=20)
        first_loc = first_loc.replace(first_path, self.sign)
        second_loc = second_loc.replace(second_path, self.sign)
        self.redirect_parser = SimilarityParser(first_loc, second_loc)
        self.redirect_parser.unquote = True
        self.redirect_parser.ignorecase = True

    # Check if redirect matches the wildcard redirect regex or the response
    # has high similarity with wildcard tested at the start
    def scan(self, path, response):
        if self.response.status == response.status == 404:
            return False

        if self.response.status != response.status:
            return True

        if self.redirect_parser and response.redirect:
            # Remove DOM (#) amd queries (?) before comparing to reduce false positives
            path = path.split("?")[0].split("#")[0]
            redirect = response.redirect.split("?")[0].split("#")[0]

            path = re.escape(unquote(path))

            regex = self.redirect_parser.regex.replace(self.sign, path)
            redirect_to_invalid = self.redirect_parser.compare(regex, redirect)

            # If redirection doesn't match the rule, mark as found
            if not redirect_to_invalid:
                return True

        # Compare 2 responses (wildcard one and given one)
        ratio = self.dynamic_parser.compareTo(response.body)

        # If the similarity ratio is high enough to proof it's wildcard
        if ratio >= self.ratio:
            return False
        elif "redirect_to_invalid" in locals() and ratio >= (self.ratio -
                                                             0.18):
            return False

        return True
Exemple #10
0
class Scanner(object):
    def __init__(self, requester, testPath=None, suffix=None):
        if testPath is None or testPath is "":
            self.testPath = RandomUtils.randString()
        else:
            self.testPath = testPath

        self.suffix = suffix if suffix is not None else ""
        self.requester = requester
        self.tester = None
        self.redirectRegExp = None
        self.invalidStatus = None
        self.dynamicParser = None
        self.ratio = 0.98
        self.redirectStatusCodes = [301, 302, 307]
        self.setup()

    def setup(self):
        firstPath = self.testPath + self.suffix
        firstResponse = self.requester.request(firstPath)
        self.invalidStatus = firstResponse.status

        if self.invalidStatus == 404:
            # Using the response status code is enough :-}
            return

        # look for redirects
        secondPath = RandomUtils.randString(omit=self.testPath) + self.suffix
        secondResponse = self.requester.request(secondPath)

        if firstResponse.status in self.redirectStatusCodes and firstResponse.redirect and secondResponse.redirect:
            self.redirectRegExp = self.generateRedirectRegExp(firstResponse.redirect, secondResponse.redirect)

        # Analyze response bodies
        self.dynamicParser = DynamicContentParser(self.requester, firstPath, firstResponse.body, secondResponse.body)

        baseRatio = float("{0:.2f}".format(self.dynamicParser.comparisonRatio))  # Rounding to 2 decimals

        # If response length is small, adjust ratio
        if len(firstResponse) < 2000:
            baseRatio -= 0.1

        if baseRatio < self.ratio:
            self.ratio = baseRatio

    def generateRedirectRegExp(self, firstLocation, secondLocation):
        if firstLocation is None or secondLocation is None:
            return None

        sm = SequenceMatcher(None, firstLocation, secondLocation)
        marks = []

        for blocks in sm.get_matching_blocks():
            i = blocks[0]
            n = blocks[2]
            # empty block

            if n == 0:
                continue

            mark = firstLocation[i:i + n]
            marks.append(mark)

        regexp = "^.*{0}.*$".format(".*".join(map(re.escape, marks)))
        return regexp

    def scan(self, path, response):
        if self.invalidStatus == 404 and response.status == 404:
            return False

        if self.invalidStatus != response.status:
            return True

        redirectToInvalid = False

        if self.redirectRegExp is not None and response.redirect is not None:
            redirectToInvalid = re.match(self.redirectRegExp, response.redirect) is not None
            # If redirection doesn't match the rule, mark as found

            if not redirectToInvalid:
                return True

        ratio = self.dynamicParser.compareTo(response.body)

        if ratio >= self.ratio:
            return False

        elif redirectToInvalid and ratio >= (self.ratio - 0.15):
            return False

        return True
Exemple #11
0
class Scanner(object):
    def __init__(self, requester, calibration=None, suffix=None, prefix=None):
        self.calibration = calibration
        self.suffix = suffix if suffix else ""
        self.prefix = prefix if prefix else ""
        self.requester = requester
        self.tester = None
        self.redirect_reg_exp = None
        self.invalid_status = None
        self.dynamic_parser = None
        self.sign = None
        self.setup()

    # Generate wildcard response information containers, this will be
    # used to compare with other path responses
    def setup(self):
        first_path = self.prefix + (
            self.calibration if self.calibration else RandomUtils.rand_string()
        ) + self.suffix
        first_response = self.requester.request(first_path)
        self.invalid_status = first_response.status

        if self.invalid_status == 404:
            # Using the response status code is enough :-}
            return

        second_path = self.prefix + (
            self.calibration if self.calibration else RandomUtils.rand_string(omit=first_path)
        ) + self.suffix
        second_response = self.requester.request(second_path)

        # Look for redirects
        if first_response.redirect and second_response.redirect:
            self.redirect_reg_exp = self.generate_redirect_reg_exp(
                first_response.redirect, first_path,
                second_response.redirect, second_path,
            )

        # Analyze response bodies
        if first_response.body is not None and second_response.body is not None:
            self.dynamic_parser = DynamicContentParser(
                self.requester, first_path, first_response.body, second_response.body
            )
        else:
            self.dynamic_parser = None

        self.ratio = float(
            "{0:.2f}".format(self.dynamic_parser.comparisonRatio)
        )  # Rounding to 2 decimals

        # The wildcard response is static
        if self.ratio == 1:
            pass
        # If response length is small, adjust ratio
        elif len(first_response) < 100:
            self.ratio -= 0.1
        elif len(first_response) < 500:
            self.ratio -= 0.05
        elif len(first_response) < 2000:
            self.ratio -= 0.02
        # If the path is reflected in response, decrease the ratio. Because
        # the difference between path lengths can reduce the similarity ratio
        if first_path in first_response.body.decode():
            self.ratio -= 0.12

    # From 2 redirects of wildcard responses, generate a regexp that matches
    # every wildcard redirect
    def generate_redirect_reg_exp(self, first_loc, first_path, second_loc, second_path):
        # Use a unique sign to locate where the path gets reflected in the redirect
        self.sign = RandomUtils.rand_string(n=20)
        first_loc = first_loc.replace(first_path, self.sign)
        second_loc = second_loc.replace(second_path, self.sign)
        reg_exp_start = "^"
        reg_exp_end = "$"

        for f, s in zip(first_loc, second_loc):
            if f == s:
                reg_exp_start += re.escape(f)
            else:
                reg_exp_start += ".*"
                break

        if reg_exp_start.endswith(".*"):
            for f, s in zip(first_loc[::-1], second_loc[::-1]):
                if f == s:
                    reg_exp_end = re.escape(f) + reg_exp_end
                else:
                    break

        return unquote(reg_exp_start + reg_exp_end)

    # Check if redirect matches the wildcard redirect regex or the response
    # has high similarity with wildcard tested at the start
    def scan(self, path, response):
        if self.invalid_status == response.status == 404:
            return False

        if self.invalid_status != response.status:
            return True

        if self.redirect_reg_exp and response.redirect:
            path = re.escape(unquote(path))
            # A lot of times, '#' or '?' will be removed in the redirect, cause false positives
            for char in ["\\#", "\\?"]:
                if char in path:
                    path = path.replace(char, "(|" + char) + ")"

            redirect_reg_exp = self.redirect_reg_exp.replace(self.sign, path)

            # Redirect sometimes encodes/decodes characters in URL, which may confuse the
            # rule check and make noise in the output, so we need to unquote() everything
            redirect_to_invalid = re.match(redirect_reg_exp, unquote(response.redirect))

            # If redirection doesn't match the rule, mark as found
            if redirect_to_invalid is None:
                return True

        # Compare 2 responses (wildcard one and given one)
        ratio = self.dynamic_parser.compareTo(response.body)

        # If the similarity ratio is high enough to proof it's wildcard
        if ratio >= self.ratio:
            return False
        elif "redirect_to_invalid" in locals() and ratio >= (self.ratio - 0.15):
            return False

        return True
Exemple #12
0
    def setup(self):
        if self.path is None or self.path is '':
            self.path = self.getRandomPath()

        firstpath_php = self.path + '.' + self.suffix[0]
        res1_php = self.requester.request(firstpath_php, True)
        secondpath_php = self.getRandomPath() + '.' + self.suffix[0]
        res2_php = self.requester.request(secondpath_php, True)

        firstpath_jsp = self.path + '.' + self.suffix[1]
        res1_jsp = self.requester.request(firstpath_jsp, True)
        secondpath_jsp = self.getRandomPath() + '.' + self.suffix[1]
        res2_jsp = self.requester.request(secondpath_jsp, True)

        firstpath_asp = self.path + '.' + self.suffix[2]
        res1_asp = self.requester.request(firstpath_asp, True)
        secondpath_asp = self.getRandomPath() + '.' + self.suffix[2]
        res2_asp = self.requester.request(secondpath_asp, True)

        if res1_asp.status_code == 404 and res1_php.status_code == 404 and res1_jsp.status_code == 404:
            self.flag = True
        else:

            if self.getHistory(
                    str(res1_php.history
                        )) in self.redirection_code and self.getHistory(
                            str(res2_php.history)) in self.redirection_code:
                regExp = self.generateRedirectRegExp(res1_php.url,
                                                     res2_php.url)
                self.redirection_regexp.append(
                    regExp) if regExp not in self.redirection_regexp else 0

            if self.getHistory(
                    str(res1_jsp.history
                        )) in self.redirection_code and self.getHistory(
                            str(res2_jsp.history)) in self.redirection_code:
                regExp = self.generateRedirectRegExp(res1_jsp.url,
                                                     res2_jsp.url)
                self.redirection_regexp.append(
                    regExp) if regExp not in self.redirection_regexp else 0

            if self.getHistory(
                    str(res1_asp.history
                        )) in self.redirection_code and self.getHistory(
                            str(res2_asp.history)) in self.redirection_code:
                regExp = self.generateRedirectRegExp(res1_asp.url,
                                                     res2_asp.url)
                self.redirection_regexp.append(
                    regExp) if regExp not in self.redirection_regexp else 0

            if res1_asp.status_code == 404 and res1_php.status_code == 404 and res1_jsp.status_code == 404:
                self.flag = True

            self.dynamic_php = DynamicContentParser(self.requester,
                                                    firstpath_php,
                                                    res1_php.text,
                                                    res2_php.text)
            if self.dynamic_php is not None:
                ratio = float('{0:.2f}'.format(
                    self.dynamic_php.comparisonRatio))
                if self.base_ratio > ratio:
                    self.base_ratio = ratio

            self.dynamic_jsp = DynamicContentParser(self.requester,
                                                    firstpath_jsp,
                                                    res1_jsp.text,
                                                    res2_jsp.text)
            if self.dynamic_jsp is not None:
                ratio = float('{0:.2f}'.format(
                    self.dynamic_jsp.comparisonRatio))
                if self.base_ratio > ratio:
                    self.base_ratio = ratio

            self.dynamic_asp = DynamicContentParser(self.requester,
                                                    firstpath_asp,
                                                    res1_asp.text,
                                                    res2_asp.text)
            if self.dynamic_asp is not None:
                ratio = float('{0:.2f}'.format(
                    self.dynamic_asp.comparisonRatio))
                if self.base_ratio > ratio:
                    self.base_ratio = ratio
Exemple #13
0
class Fuzzer(object):
    def __init__(self, requester, path=None):
        self.requester = requester
        self.path = path
        self.suffix = ['php', 'jsp', 'asp']
        self.redirection_code = ['301', '302', '303', '307']
        self.base_ratio = 0.98
        self.flag = False
        self.redirection_regexp = []
        self.setup()

    def getRandomPath(self):
        letters = string.ascii_letters + string.digits
        return ''.join(random.choice(letters) for i in range(8))

    def generateRedirectRegExp(self, firstLocation, secondLocation):
        if firstLocation is None or secondLocation is None:
            return None
        sm = SequenceMatcher(None, firstLocation, secondLocation)
        marks = []
        for blocks in sm.get_matching_blocks():
            i = blocks[0]
            n = blocks[2]
            # empty block
            if n == 0:
                continue
            mark = firstLocation[i:i + n]
            if mark.startswith('http') or mark.startswith('https'):
                marks.append(mark)
        regexp = "^.*{0}.*$".format(".*".join(map(re.escape, marks))).replace(
            'http', '(https|http)')
        return regexp

    def getDmain(self, url):
        url_parser = urllib.parse.urlparse(url)
        return url_parser.scheme + '://' + url_parser.netloc

    def getHistory(self, history):
        history = re.findall('\d+', history)
        history = history[0] if len(history) >= 1 else []
        return str(history)

    def setup(self):
        if self.path is None or self.path is '':
            self.path = self.getRandomPath()

        firstpath_php = self.path + '.' + self.suffix[0]
        res1_php = self.requester.request(firstpath_php, True)
        secondpath_php = self.getRandomPath() + '.' + self.suffix[0]
        res2_php = self.requester.request(secondpath_php, True)

        firstpath_jsp = self.path + '.' + self.suffix[1]
        res1_jsp = self.requester.request(firstpath_jsp, True)
        secondpath_jsp = self.getRandomPath() + '.' + self.suffix[1]
        res2_jsp = self.requester.request(secondpath_jsp, True)

        firstpath_asp = self.path + '.' + self.suffix[2]
        res1_asp = self.requester.request(firstpath_asp, True)
        secondpath_asp = self.getRandomPath() + '.' + self.suffix[2]
        res2_asp = self.requester.request(secondpath_asp, True)

        if res1_asp.status_code == 404 and res1_php.status_code == 404 and res1_jsp.status_code == 404:
            self.flag = True
        else:

            if self.getHistory(
                    str(res1_php.history
                        )) in self.redirection_code and self.getHistory(
                            str(res2_php.history)) in self.redirection_code:
                regExp = self.generateRedirectRegExp(res1_php.url,
                                                     res2_php.url)
                self.redirection_regexp.append(
                    regExp) if regExp not in self.redirection_regexp else 0

            if self.getHistory(
                    str(res1_jsp.history
                        )) in self.redirection_code and self.getHistory(
                            str(res2_jsp.history)) in self.redirection_code:
                regExp = self.generateRedirectRegExp(res1_jsp.url,
                                                     res2_jsp.url)
                self.redirection_regexp.append(
                    regExp) if regExp not in self.redirection_regexp else 0

            if self.getHistory(
                    str(res1_asp.history
                        )) in self.redirection_code and self.getHistory(
                            str(res2_asp.history)) in self.redirection_code:
                regExp = self.generateRedirectRegExp(res1_asp.url,
                                                     res2_asp.url)
                self.redirection_regexp.append(
                    regExp) if regExp not in self.redirection_regexp else 0

            if res1_asp.status_code == 404 and res1_php.status_code == 404 and res1_jsp.status_code == 404:
                self.flag = True

            self.dynamic_php = DynamicContentParser(self.requester,
                                                    firstpath_php,
                                                    res1_php.text,
                                                    res2_php.text)
            if self.dynamic_php is not None:
                ratio = float('{0:.2f}'.format(
                    self.dynamic_php.comparisonRatio))
                if self.base_ratio > ratio:
                    self.base_ratio = ratio

            self.dynamic_jsp = DynamicContentParser(self.requester,
                                                    firstpath_jsp,
                                                    res1_jsp.text,
                                                    res2_jsp.text)
            if self.dynamic_jsp is not None:
                ratio = float('{0:.2f}'.format(
                    self.dynamic_jsp.comparisonRatio))
                if self.base_ratio > ratio:
                    self.base_ratio = ratio

            self.dynamic_asp = DynamicContentParser(self.requester,
                                                    firstpath_asp,
                                                    res1_asp.text,
                                                    res2_asp.text)
            if self.dynamic_asp is not None:
                ratio = float('{0:.2f}'.format(
                    self.dynamic_asp.comparisonRatio))
                if self.base_ratio > ratio:
                    self.base_ratio = ratio

    def fuzz(self, cmp_page):
        if self.flag == True:
            if cmp_page.status_code == 404:
                return False
            else:
                return True
        else:
            if cmp_page.status_code == 404:
                return False
            redirectToInvalid = []
            for express in self.redirection_regexp:
                if express is not None:
                    redirectToInvalid.append(
                        re.match(express, cmp_page.url) is not None)
            if not any(redirectToInvalid):
                return True
            ratio_php = self.dynamic_php.compareTo(cmp_page.text)
            ratio_jsp = self.dynamic_jsp.compareTo(cmp_page.text)
            ratio_asp = self.dynamic_asp.compareTo(cmp_page.text)
            if self.base_ratio <= ratio_php or self.base_ratio <= ratio_jsp or self.base_ratio <= ratio_asp:
                return False
            elif any(redirectToInvalid) and (
                (self.base_ratio - 0.15) <= ratio_php or
                (self.base_ratio - 0.15) <= ratio_jsp or
                (self.base_ratio - 0.15) <= ratio_asp):
                return False
            return True