def setup(self):
for path in self.testPath:
firstPath = RandomUtils.randString() + '/' + path + self.suffix
firstResponse = self.requester.request(firstPath)
if firstResponse.status not in self.invalidStatus:
self.invalidStatus.append(firstResponse.status)
if firstResponse.status == 404:
# Using the response status code is enough :-}
continue
# look for redirects
secondPath = RandomUtils.randString() + '/' + path + self.suffix
secondResponse = self.requester.request(secondPath)
if firstResponse.status in self.redirectStatusCodes and firstResponse.redirect and secondResponse.redirect:
self.redirectRegExp.append(
self.generateRedirectRegExp(firstResponse.redirect,
secondResponse.redirect))
# Analyze response bodies
dynamicParser = DynamicContentParser(self.requester, firstPath,
firstResponse.body,
secondResponse.body)
baseRatio = float("{0:.2f}".format(
dynamicParser.comparisonRatio)) # Rounding to 2 decimals
ratio = self.ratio
# If response length is small, adjust ratio
if len(firstResponse) < 2000:
baseRatio -= 0.1
if baseRatio < self.ratio:
ratio = baseRatio
if self.dynamicParser:
flag = 0
for _dynamicParser, __ in self.dynamicParser:
_ratio = dynamicParser.compareTo(_dynamicParser.cleanPage)
flag += _ratio > ratio
if not flag:
self.dynamicParser.append((dynamicParser, ratio))
else:
self.dynamicParser.append((dynamicParser, ratio))
class Scanner(object):
def __init__(self, requester, testPath=None, suffix=None, preffix=None):
if testPath is None or testPath == "":
self.testPath = RandomUtils.randString()
else:
self.testPath = testPath
self.suffix = suffix if suffix else ""
self.preffix = preffix if preffix else ""
self.requester = requester
self.tester = None
self.redirectRegExp = None
self.invalidStatus = None
self.dynamicParser = None
self.ratio = 0.98
self.redirectStatusCodes = [301, 302, 303, 307, 308]
self.setup()
def setup(self):
firstPath = self.preffix + self.testPath + self.suffix
firstResponse = self.requester.request(firstPath)
self.invalidStatus = firstResponse.status
secondPath = self.preffix + RandomUtils.randString(
omit=self.testPath) + self.suffix
secondResponse = self.requester.request(secondPath)
if self.invalidStatus == 404:
# Using the response status code is enough :-}
return
# look for redirects
elif firstResponse.status in self.redirectStatusCodes and firstResponse.redirect and secondResponse.redirect:
self.redirectRegExp = self.generateRedirectRegExp(
firstResponse.redirect, secondResponse.redirect)
# Analyze response bodies
self.dynamicParser = DynamicContentParser(self.requester, firstPath,
firstResponse.body,
secondResponse.body)
baseRatio = float("{0:.2f}".format(
self.dynamicParser.comparisonRatio)) # Rounding to 2 decimals
# If response length is small, adjust ratio
if len(firstResponse) < 2000:
baseRatio -= 0.1
if baseRatio < self.ratio:
self.ratio = baseRatio
def generateRedirectRegExp(self, firstLocation, secondLocation):
if firstLocation is None or secondLocation is None:
return None
sm = SequenceMatcher(None, firstLocation, secondLocation)
marks = []
for blocks in sm.get_matching_blocks():
i = blocks[0]
n = blocks[2]
# empty block
if n == 0:
continue
mark = firstLocation[i:i + n]
marks.append(mark)
regexp = "^.*{0}.*$".format(".*".join(map(re.escape, marks)))
return regexp
def scan(self, path, response):
if self.invalidStatus == response.status == 404:
return False
if self.invalidStatus != response.status:
return True
redirectToInvalid = False
if self.redirectRegExp and response.redirect:
redirectToInvalid = (re.match(self.redirectRegExp,
response.redirect) is not None)
# If redirection doesn't match the rule, mark as found
if not redirectToInvalid:
return True
ratio = self.dynamicParser.compareTo(response.body)
if ratio >= self.ratio:
return False
elif redirectToInvalid and ratio >= (self.ratio - 0.15):
return False
return True
class Scanner(object):
def __init__(self, requester, calibration=None, suffix=None, prefix=None):
self.calibration = calibration
self.suffix = suffix if suffix else ""
self.prefix = prefix if prefix else ""
self.requester = requester
self.tester = None
self.redirectRegExp = None
self.invalidStatus = None
self.dynamicParser = None
self.sign = None
self.ratio = 0.98
self.setup()
def setup(self):
firstPath = self.prefix + (self.calibration if self.calibration else
RandomUtils.randString()) + self.suffix
firstResponse = self.requester.request(firstPath)
self.invalidStatus = firstResponse.status
if self.invalidStatus == 404:
# Using the response status code is enough :-}
return
secondPath = self.prefix + (self.calibration if self.calibration else
RandomUtils.randString(
omit=firstPath)) + self.suffix
secondResponse = self.requester.request(secondPath)
# Look for redirects
if firstResponse.redirect and secondResponse.redirect:
self.redirectRegExp = self.generateRedirectRegExp(
firstResponse.redirect, secondResponse.redirect)
# Analyze response bodies
if firstResponse.body is not None and secondResponse.body is not None:
self.dynamicParser = DynamicContentParser(self.requester,
firstPath,
firstResponse.body,
secondResponse.body)
else:
self.dynamicParser = None
baseRatio = float("{0:.2f}".format(
self.dynamicParser.comparisonRatio)) # Rounding to 2 decimals
# If response length is small, adjust ratio
if len(firstResponse) < 2000:
baseRatio -= 0.1
if baseRatio < self.ratio:
self.ratio = baseRatio
def generateRedirectRegExp(self, firstLoc, firstPath, secondLoc,
secondPath):
# Use a unique sign to locate where the path gets reflected in the redirect
self.sign = RandomUtils.randString(n=20)
firstLoc = firstLoc.replace(firstPath, self.sign)
secondLoc = secondLoc.replace(secondPath, self.sign)
regExpStart = "^"
regExpEnd = "$"
for f, s in zip(firstLoc, secondLoc):
if f == s:
regExpStart += re.escape(f)
else:
regExpStart += ".*"
break
if regExpStart.endswith(".*"):
for f, s in zip(firstLoc[::-1], secondLoc[::-1]):
if f == s:
regExpEnd = re.escape(f) + regExpEnd
else:
break
return unquote(regExpStart + regExpEnd)
def scan(self, path, response):
if self.invalidStatus == response.status == 404:
return False
if self.invalidStatus != response.status:
return True
if self.redirectRegExp and response.redirect:
path = re.escape(unquote(path))
# A lot of times, '#' or '?' will be removed in the redirect, cause false positives
for char in ["\\#", "\\?"]:
if char in path:
path = path.replace(char, "(|" + char) + ")"
redirectRegExp = self.redirectRegExp.replace(self.sign, path)
# Redirect sometimes encodes/decodes characters in URL, which may confuse the
# rule check and make noise in the output, so we need to unquote() everything
redirectToInvalid = re.match(redirectRegExp,
unquote(response.redirect))
# If redirection doesn't match the rule, mark as found
if redirectToInvalid is None:
return True
ratio = self.dynamicParser.compareTo(response.body)
if ratio >= self.ratio:
return False
elif "redirectToInvalid" in locals() and ratio >= (self.ratio - 0.15):
return False
return True
class Scanner(object):
def __init__(self,
requester,
calibration=None,
suffix=None,
prefix=None,
tested=None):
self.calibration = calibration
self.suffix = suffix if suffix else ""
self.prefix = prefix if prefix else ""
self.tested = tested
self.requester = requester
self.tester = None
self.response = None
self.dynamic_parser = None
self.redirect_parser = None
self.sign = None
self.setup()
def duplicate(self, response):
if not self.tested:
return
for t in self.tested:
for tester in self.tested[t].values():
if [response.status, response.body, response.redirect] == [
tester.response.status, tester.response.body,
tester.response.redirect
]:
return tester
return
"""
Generate wildcard response information containers, this will be
used to compare with other path responses
"""
def setup(self):
first_path = self.prefix + (self.calibration if self.calibration else
rand_string()) + self.suffix
first_response = self.requester.request(first_path)
self.response = first_response
if self.response.status == 404:
# Using the response status code is enough :-}
return
duplicate = self.duplicate(first_response)
if duplicate:
# Another test had been performed and shows the same response as this
self.ratio = duplicate.ratio
self.dynamic_parser = duplicate.dynamic_parser
self.redirect_parser = duplicate.redirect_parser
self.sign = duplicate.sign
return
second_path = self.prefix + (self.calibration
if self.calibration else rand_string(
omit=first_path)) + self.suffix
second_response = self.requester.request(second_path)
if first_response.redirect and second_response.redirect:
self.generate_redirect_reg_exp(
first_response.redirect,
first_path,
second_response.redirect,
second_path,
)
# Analyze response bodies
if first_response.body is not None and second_response.body is not None:
self.dynamic_parser = DynamicContentParser(self.requester,
first_path,
first_response.body,
second_response.body)
else:
self.dynamic_parser = None
self.ratio = float("{0:.2f}".format(
self.dynamic_parser.comparisonRatio)) # Rounding to 2 decimals
# The wildcard response is static
if self.ratio == 1:
pass
# Adjusting ratio based on response length
elif len(first_response) < 100:
self.ratio -= 0.1
elif len(first_response) < 500:
self.ratio -= 0.05
elif len(first_response) < 2000:
self.ratio -= 0.02
else:
self.ratio -= 0.01
"""
If the path is reflected in response, decrease the ratio. Because
the difference between path lengths can reduce the similarity ratio
"""
if first_path in first_response.body.decode():
if len(first_response) < 200:
self.ratio -= 0.15 + 15 / len(first_response)
elif len(first_response) < 800:
self.ratio -= 0.06 + 30 / len(first_response)
elif len(first_response) < 5000:
self.ratio -= 0.03 + 80 / len(first_response)
elif len(first_response) < 20000:
self.ratio -= 0.02 + 200 / len(first_response)
else:
self.ratio -= 0.01
"""
From 2 redirects of wildcard responses, generate a regexp that matches
every wildcard redirect
"""
def generate_redirect_reg_exp(self, first_loc, first_path, second_loc,
second_path):
# Use a unique sign to locate where the path gets reflected in the redirect
self.sign = rand_string(n=20)
first_loc = first_loc.replace(first_path, self.sign)
second_loc = second_loc.replace(second_path, self.sign)
self.redirect_parser = SimilarityParser(first_loc, second_loc)
self.redirect_parser.unquote = True
self.redirect_parser.ignorecase = True
# Check if redirect matches the wildcard redirect regex or the response
# has high similarity with wildcard tested at the start
def scan(self, path, response):
if self.response.status == response.status == 404:
return False
if self.response.status != response.status:
return True
if self.redirect_parser and response.redirect:
# Remove DOM (#) amd queries (?) before comparing to reduce false positives
path = path.split("?")[0].split("#")[0]
redirect = response.redirect.split("?")[0].split("#")[0]
path = re.escape(unquote(path))
regex = self.redirect_parser.regex.replace(self.sign, path)
redirect_to_invalid = self.redirect_parser.compare(regex, redirect)
# If redirection doesn't match the rule, mark as found
if not redirect_to_invalid:
return True
# Compare 2 responses (wildcard one and given one)
ratio = self.dynamic_parser.compareTo(response.body)
# If the similarity ratio is high enough to proof it's wildcard
if ratio >= self.ratio:
return False
elif "redirect_to_invalid" in locals() and ratio >= (self.ratio -
0.18):
return False
return True
class Scanner(object):
def __init__(self, requester, testPath=None, suffix=None):
if testPath is None or testPath is "":
self.testPath = RandomUtils.randString()
else:
self.testPath = testPath
self.suffix = suffix if suffix is not None else ""
self.requester = requester
self.tester = None
self.redirectRegExp = None
self.invalidStatus = None
self.dynamicParser = None
self.ratio = 0.98
self.redirectStatusCodes = [301, 302, 307]
self.setup()
def setup(self):
firstPath = self.testPath + self.suffix
firstResponse = self.requester.request(firstPath)
self.invalidStatus = firstResponse.status
if self.invalidStatus == 404:
# Using the response status code is enough :-}
return
# look for redirects
secondPath = RandomUtils.randString(omit=self.testPath) + self.suffix
secondResponse = self.requester.request(secondPath)
if firstResponse.status in self.redirectStatusCodes and firstResponse.redirect and secondResponse.redirect:
self.redirectRegExp = self.generateRedirectRegExp(firstResponse.redirect, secondResponse.redirect)
# Analyze response bodies
self.dynamicParser = DynamicContentParser(self.requester, firstPath, firstResponse.body, secondResponse.body)
baseRatio = float("{0:.2f}".format(self.dynamicParser.comparisonRatio)) # Rounding to 2 decimals
# If response length is small, adjust ratio
if len(firstResponse) < 2000:
baseRatio -= 0.1
if baseRatio < self.ratio:
self.ratio = baseRatio
def generateRedirectRegExp(self, firstLocation, secondLocation):
if firstLocation is None or secondLocation is None:
return None
sm = SequenceMatcher(None, firstLocation, secondLocation)
marks = []
for blocks in sm.get_matching_blocks():
i = blocks[0]
n = blocks[2]
# empty block
if n == 0:
continue
mark = firstLocation[i:i + n]
marks.append(mark)
regexp = "^.*{0}.*$".format(".*".join(map(re.escape, marks)))
return regexp
def scan(self, path, response):
if self.invalidStatus == 404 and response.status == 404:
return False
if self.invalidStatus != response.status:
return True
redirectToInvalid = False
if self.redirectRegExp is not None and response.redirect is not None:
redirectToInvalid = re.match(self.redirectRegExp, response.redirect) is not None
# If redirection doesn't match the rule, mark as found
if not redirectToInvalid:
return True
ratio = self.dynamicParser.compareTo(response.body)
if ratio >= self.ratio:
return False
elif redirectToInvalid and ratio >= (self.ratio - 0.15):
return False
return True
class Scanner(object):
def __init__(self, requester, calibration=None, suffix=None, prefix=None):
self.calibration = calibration
self.suffix = suffix if suffix else ""
self.prefix = prefix if prefix else ""
self.requester = requester
self.tester = None
self.redirect_reg_exp = None
self.invalid_status = None
self.dynamic_parser = None
self.sign = None
self.setup()
# Generate wildcard response information containers, this will be
# used to compare with other path responses
def setup(self):
first_path = self.prefix + (
self.calibration if self.calibration else RandomUtils.rand_string()
) + self.suffix
first_response = self.requester.request(first_path)
self.invalid_status = first_response.status
if self.invalid_status == 404:
# Using the response status code is enough :-}
return
second_path = self.prefix + (
self.calibration if self.calibration else RandomUtils.rand_string(omit=first_path)
) + self.suffix
second_response = self.requester.request(second_path)
# Look for redirects
if first_response.redirect and second_response.redirect:
self.redirect_reg_exp = self.generate_redirect_reg_exp(
first_response.redirect, first_path,
second_response.redirect, second_path,
)
# Analyze response bodies
if first_response.body is not None and second_response.body is not None:
self.dynamic_parser = DynamicContentParser(
self.requester, first_path, first_response.body, second_response.body
)
else:
self.dynamic_parser = None
self.ratio = float(
"{0:.2f}".format(self.dynamic_parser.comparisonRatio)
) # Rounding to 2 decimals
# The wildcard response is static
if self.ratio == 1:
pass
# If response length is small, adjust ratio
elif len(first_response) < 100:
self.ratio -= 0.1
elif len(first_response) < 500:
self.ratio -= 0.05
elif len(first_response) < 2000:
self.ratio -= 0.02
# If the path is reflected in response, decrease the ratio. Because
# the difference between path lengths can reduce the similarity ratio
if first_path in first_response.body.decode():
self.ratio -= 0.12
# From 2 redirects of wildcard responses, generate a regexp that matches
# every wildcard redirect
def generate_redirect_reg_exp(self, first_loc, first_path, second_loc, second_path):
# Use a unique sign to locate where the path gets reflected in the redirect
self.sign = RandomUtils.rand_string(n=20)
first_loc = first_loc.replace(first_path, self.sign)
second_loc = second_loc.replace(second_path, self.sign)
reg_exp_start = "^"
reg_exp_end = "$"
for f, s in zip(first_loc, second_loc):
if f == s:
reg_exp_start += re.escape(f)
else:
reg_exp_start += ".*"
break
if reg_exp_start.endswith(".*"):
for f, s in zip(first_loc[::-1], second_loc[::-1]):
if f == s:
reg_exp_end = re.escape(f) + reg_exp_end
else:
break
return unquote(reg_exp_start + reg_exp_end)
# Check if redirect matches the wildcard redirect regex or the response
# has high similarity with wildcard tested at the start
def scan(self, path, response):
if self.invalid_status == response.status == 404:
return False
if self.invalid_status != response.status:
return True
if self.redirect_reg_exp and response.redirect:
path = re.escape(unquote(path))
# A lot of times, '#' or '?' will be removed in the redirect, cause false positives
for char in ["\\#", "\\?"]:
if char in path:
path = path.replace(char, "(|" + char) + ")"
redirect_reg_exp = self.redirect_reg_exp.replace(self.sign, path)
# Redirect sometimes encodes/decodes characters in URL, which may confuse the
# rule check and make noise in the output, so we need to unquote() everything
redirect_to_invalid = re.match(redirect_reg_exp, unquote(response.redirect))
# If redirection doesn't match the rule, mark as found
if redirect_to_invalid is None:
return True
# Compare 2 responses (wildcard one and given one)
ratio = self.dynamic_parser.compareTo(response.body)
# If the similarity ratio is high enough to proof it's wildcard
if ratio >= self.ratio:
return False
elif "redirect_to_invalid" in locals() and ratio >= (self.ratio - 0.15):
return False
return True
class Fuzzer(object):
def __init__(self, requester, path=None):
self.requester = requester
self.path = path
self.suffix = ['php', 'jsp', 'asp']
self.redirection_code = ['301', '302', '303', '307']
self.base_ratio = 0.98
self.flag = False
self.redirection_regexp = []
self.setup()
def getRandomPath(self):
letters = string.ascii_letters + string.digits
return ''.join(random.choice(letters) for i in range(8))
def generateRedirectRegExp(self, firstLocation, secondLocation):
if firstLocation is None or secondLocation is None:
return None
sm = SequenceMatcher(None, firstLocation, secondLocation)
marks = []
for blocks in sm.get_matching_blocks():
i = blocks[0]
n = blocks[2]
# empty block
if n == 0:
continue
mark = firstLocation[i:i + n]
if mark.startswith('http') or mark.startswith('https'):
marks.append(mark)
regexp = "^.*{0}.*$".format(".*".join(map(re.escape, marks))).replace(
'http', '(https|http)')
return regexp
def getDmain(self, url):
url_parser = urllib.parse.urlparse(url)
return url_parser.scheme + '://' + url_parser.netloc
def getHistory(self, history):
history = re.findall('\d+', history)
history = history[0] if len(history) >= 1 else []
return str(history)
def setup(self):
if self.path is None or self.path is '':
self.path = self.getRandomPath()
firstpath_php = self.path + '.' + self.suffix[0]
res1_php = self.requester.request(firstpath_php, True)
secondpath_php = self.getRandomPath() + '.' + self.suffix[0]
res2_php = self.requester.request(secondpath_php, True)
firstpath_jsp = self.path + '.' + self.suffix[1]
res1_jsp = self.requester.request(firstpath_jsp, True)
secondpath_jsp = self.getRandomPath() + '.' + self.suffix[1]
res2_jsp = self.requester.request(secondpath_jsp, True)
firstpath_asp = self.path + '.' + self.suffix[2]
res1_asp = self.requester.request(firstpath_asp, True)
secondpath_asp = self.getRandomPath() + '.' + self.suffix[2]
res2_asp = self.requester.request(secondpath_asp, True)
if res1_asp.status_code == 404 and res1_php.status_code == 404 and res1_jsp.status_code == 404:
self.flag = True
else:
if self.getHistory(
str(res1_php.history
)) in self.redirection_code and self.getHistory(
str(res2_php.history)) in self.redirection_code:
regExp = self.generateRedirectRegExp(res1_php.url,
res2_php.url)
self.redirection_regexp.append(
regExp) if regExp not in self.redirection_regexp else 0
if self.getHistory(
str(res1_jsp.history
)) in self.redirection_code and self.getHistory(
str(res2_jsp.history)) in self.redirection_code:
regExp = self.generateRedirectRegExp(res1_jsp.url,
res2_jsp.url)
self.redirection_regexp.append(
regExp) if regExp not in self.redirection_regexp else 0
if self.getHistory(
str(res1_asp.history
)) in self.redirection_code and self.getHistory(
str(res2_asp.history)) in self.redirection_code:
regExp = self.generateRedirectRegExp(res1_asp.url,
res2_asp.url)
self.redirection_regexp.append(
regExp) if regExp not in self.redirection_regexp else 0
if res1_asp.status_code == 404 and res1_php.status_code == 404 and res1_jsp.status_code == 404:
self.flag = True
self.dynamic_php = DynamicContentParser(self.requester,
firstpath_php,
res1_php.text,
res2_php.text)
if self.dynamic_php is not None:
ratio = float('{0:.2f}'.format(
self.dynamic_php.comparisonRatio))
if self.base_ratio > ratio:
self.base_ratio = ratio
self.dynamic_jsp = DynamicContentParser(self.requester,
firstpath_jsp,
res1_jsp.text,
res2_jsp.text)
if self.dynamic_jsp is not None:
ratio = float('{0:.2f}'.format(
self.dynamic_jsp.comparisonRatio))
if self.base_ratio > ratio:
self.base_ratio = ratio
self.dynamic_asp = DynamicContentParser(self.requester,
firstpath_asp,
res1_asp.text,
res2_asp.text)
if self.dynamic_asp is not None:
ratio = float('{0:.2f}'.format(
self.dynamic_asp.comparisonRatio))
if self.base_ratio > ratio:
self.base_ratio = ratio
def fuzz(self, cmp_page):
if self.flag == True:
if cmp_page.status_code == 404:
return False
else:
return True
else:
if cmp_page.status_code == 404:
return False
redirectToInvalid = []
for express in self.redirection_regexp:
if express is not None:
redirectToInvalid.append(
re.match(express, cmp_page.url) is not None)
if not any(redirectToInvalid):
return True
ratio_php = self.dynamic_php.compareTo(cmp_page.text)
ratio_jsp = self.dynamic_jsp.compareTo(cmp_page.text)
ratio_asp = self.dynamic_asp.compareTo(cmp_page.text)
if self.base_ratio <= ratio_php or self.base_ratio <= ratio_jsp or self.base_ratio <= ratio_asp:
return False
elif any(redirectToInvalid) and (
(self.base_ratio - 0.15) <= ratio_php or
(self.base_ratio - 0.15) <= ratio_jsp or
(self.base_ratio - 0.15) <= ratio_asp):
return False
return True