def setup(self):
first_path = self.prefix + (self.calibration if self.calibration else
RandomUtils.rand_string()) + self.suffix
first_response = self.requester.request(first_path)
self.invalid_status = first_response.status
if self.invalid_status == 404:
# Using the response status code is enough :-}
return
second_path = self.prefix + (self.calibration if self.calibration else
RandomUtils.rand_string(
omit=first_path)) + self.suffix
second_response = self.requester.request(second_path)
# Look for redirects
if first_response.redirect and second_response.redirect:
self.redirect_reg_exp = self.generate_redirect_reg_exp(
first_response.redirect,
first_path,
second_response.redirect,
second_path,
)
# Analyze response bodies
if first_response.body is not None and second_response.body is not None:
self.dynamic_parser = DynamicContentParser(self.requester,
first_path,
first_response.body,
second_response.body)
else:
self.dynamic_parser = None
self.ratio = float("{0:.2f}".format(
self.dynamic_parser.comparisonRatio)) # Rounding to 2 decimals
# The wildcard response is static
if self.ratio == 1:
pass
# Adjusting ratio based on response length
elif len(first_response) < 100:
self.ratio -= 0.1
elif len(first_response) < 500:
self.ratio -= 0.05
elif len(first_response) < 2000:
self.ratio -= 0.02
else:
self.ratio -= 0.01
# If the path is reflected in response, decrease the ratio. Because
# the difference between path lengths can reduce the similarity ratio
if first_path in first_response.body.decode(
) and len(first_response) < 100000:
self.ratio -= 0.1
def setup(self):
firstPath = self.preffix + self.testPath + self.suffix
firstResponse = self.requester.request(firstPath)
self.invalidStatus = firstResponse.status
secondPath = self.preffix + RandomUtils.randString(
omit=self.testPath) + self.suffix
secondResponse = self.requester.request(secondPath)
if self.invalidStatus == 404:
# Using the response status code is enough :-}
return
# look for redirects
elif firstResponse.status in self.redirectStatusCodes and firstResponse.redirect and secondResponse.redirect:
self.redirectRegExp = self.generateRedirectRegExp(
firstResponse.redirect, secondResponse.redirect)
# Analyze response bodies
self.dynamicParser = DynamicContentParser(self.requester, firstPath,
firstResponse.body,
secondResponse.body)
baseRatio = float("{0:.2f}".format(
self.dynamicParser.comparisonRatio)) # Rounding to 2 decimals
# If response length is small, adjust ratio
if len(firstResponse) < 2000:
baseRatio -= 0.1
if baseRatio < self.ratio:
self.ratio = baseRatio
def setup(self):
firstPath = self.testPath + self.suffix
firstResponse = self.requester.request(firstPath)
self.invalidStatus = firstResponse.status
if self.invalidStatus == 404:
# Using the response status code is enough :-}
return
# look for redirects
secondPath = RandomUtils.randString(omit=self.testPath) + self.suffix
secondResponse = self.requester.request(secondPath)
if firstResponse.status in self.redirectStatusCodes and firstResponse.redirect and secondResponse.redirect:
self.redirectRegExp = self.generateRedirectRegExp(firstResponse.redirect, secondResponse.redirect)
# Analyze response bodies
self.dynamicParser = DynamicContentParser(self.requester, firstPath, firstResponse.body, secondResponse.body)
baseRatio = float("{0:.2f}".format(self.dynamicParser.comparisonRatio)) # Rounding to 2 decimals
# If response length is small, adjust ratio
if len(firstResponse) < 2000:
baseRatio -= 0.1
if baseRatio < self.ratio:
self.ratio = baseRatio
def setup(self):
for path in self.testPath:
firstPath = RandomUtils.randString() + '/' + path + self.suffix
firstResponse = self.requester.request(firstPath)
if firstResponse.status not in self.invalidStatus:
self.invalidStatus.append(firstResponse.status)
if firstResponse.status == 404:
# Using the response status code is enough :-}
continue
# look for redirects
secondPath = RandomUtils.randString() + '/' + path + self.suffix
secondResponse = self.requester.request(secondPath)
if firstResponse.status in self.redirectStatusCodes and firstResponse.redirect and secondResponse.redirect:
self.redirectRegExp.append(
self.generateRedirectRegExp(firstResponse.redirect,
secondResponse.redirect))
# Analyze response bodies
dynamicParser = DynamicContentParser(self.requester, firstPath,
firstResponse.body,
secondResponse.body)
baseRatio = float("{0:.2f}".format(
dynamicParser.comparisonRatio)) # Rounding to 2 decimals
ratio = self.ratio
# If response length is small, adjust ratio
if len(firstResponse) < 2000:
baseRatio -= 0.1
if baseRatio < self.ratio:
ratio = baseRatio
if self.dynamicParser:
flag = 0
for _dynamicParser, __ in self.dynamicParser:
_ratio = dynamicParser.compareTo(_dynamicParser.cleanPage)
flag += _ratio > ratio
if not flag:
self.dynamicParser.append((dynamicParser, ratio))
else:
self.dynamicParser.append((dynamicParser, ratio))
def get_special_str(self, str):
str_list = list(str)
str_set_len = len(set(str_list))
for _ in range(100):
if str_set_len == 1 and len(str_list) == 1:
new_str = RandomUtils.randString(n=1, omit=str_list)
elif str_set_len == 1:
_ = RandomUtils.randString(n=len(str_list) - str_set_len)
new_str = '{}{}'.format(str_list[0], _)
else:
random.shuffle(str_list)
new_str = ''.join(str_list)
if new_str == str:
continue
return new_str
else:
raise Exception('Special Str: {}'.format(str))
def setup(self):
first_path = self.prefix + (
self.calibration if self.calibration else RandomUtils.rand_string()
) + self.suffix
first_response = self.requester.request(first_path)
self.invalid_status = first_response.status
if self.invalid_status == 404:
# Using the response status code is enough :-}
return
second_path = self.prefix + (
self.calibration if self.calibration else RandomUtils.rand_string(omit=first_path)
) + self.suffix
second_response = self.requester.request(second_path)
# Look for redirects
if first_response.redirect and second_response.redirect:
self.redirect_reg_exp = self.generate_redirect_reg_exp(
first_response.redirect, first_path,
second_response.redirect, second_path,
)
# Analyze response bodies
if first_response.body is not None and second_response.body is not None:
self.dynamic_parser = DynamicContentParser(
self.requester, first_path, first_response.body, second_response.body
)
else:
self.dynamic_parser = None
base_ratio = float(
"{0:.2f}".format(self.dynamic_parser.comparisonRatio)
) # Rounding to 2 decimals
# If response length is small, adjust ratio
if len(first_response) < 500:
base_ratio -= 0.15
elif len(first_response) < 2000:
base_ratio -= 0.1
if base_ratio < self.ratio:
self.ratio = base_ratio
def get_special_path(self, path):
special_path = []
if path.endswith('/'):
_origin_path, _dir = path[:-1].rsplit('/', 1)
else:
_origin_path, _last_element = path.rsplit('/', 1)
if path.endswith('/'):
# unzip exception
_origin_path, _dir = path[:-1].rsplit('/', 1)
special_dir = self.get_special_str(_dir)
_path = '{}/{}/'.format(_origin_path, special_dir)
if not _path.startswith('/'):
_path = '/' + _path
special_path.append(_path)
elif '.' in _last_element:
_origin_path, _filename_ext = path.rsplit('/', 1)
_filename, _ext = _filename_ext.rsplit('.', 1)
# special filename
if not _filename:
_filename = RandomUtils.randString(n=8)
special_filename = self.get_special_str(_filename)
_path = '{}/{}.{}'.format(_origin_path, special_filename, _ext)
if not _path.startswith('/'):
_path = '/' + _path
special_path.append(_path)
# special extension
if not _ext:
_ext = RandomUtils.randString(n=8)
special_extension = self.get_special_str(_ext)
_path = '{}/{}.{}'.format(_origin_path, _filename,
special_extension)
if not _path.startswith('/'):
_path = '/' + _path
special_path.append(_path)
else:
_origin_path, _filename = path.rsplit('/', 1)
special_filename = self.get_special_str(_filename)
_path = '{}/{}'.format(_origin_path, special_filename)
if not _path.startswith('/'):
_path = '/' + _path
special_path.append(_path)
return special_path
def __init__(self, requester, testPath=None, suffix=None):
if testPath is None or testPath is "":
self.testPath = RandomUtils.randString()
else:
self.testPath = testPath
self.suffix = suffix if suffix is not None else ""
self.requester = requester
self.tester = None
self.redirectRegExp = None
self.invalidStatus = None
self.dynamicParser = None
self.ratio = 0.98
self.redirectStatusCodes = [301, 302, 307]
self.setup()
def __init__(self, requester, testPath=None, suffix=None):
if testPath is None or testPath is "":
self.testPath = RandomUtils.randString()
else:
self.testPath = testPath
self.suffix = suffix if suffix is not None else ""
self.requester = requester
self.tester = None
self.redirectRegExp = None
self.invalidStatus = None
self.dynamicParser = None
self.ratio = 0.98
self.redirectStatusCodes = [301, 302, 307]
self.setup()
def __init__(self, requester, testPath=None, suffix=None, preffix=None):
if not testPath:
self.testPath = RandomUtils.randString()
else:
self.testPath = testPath
self.suffix = suffix if suffix else ""
self.preffix = preffix if preffix else ""
self.requester = requester
self.tester = None
self.redirectRegExp = None
self.invalidStatus = None
self.dynamicParser = None
self.ratio = 0.98
self.setup()
def generate_redirect_reg_exp(self, first_loc, first_path, second_loc, second_path):
# Use a unique sign to locate where the path gets reflected in the redirect
self.sign = RandomUtils.rand_string(n=20)
first_loc = first_loc.replace(first_path, self.sign)
second_loc = second_loc.replace(second_path, self.sign)
reg_exp_start = "^"
reg_exp_end = "$"
for f, s in zip(first_loc, second_loc):
if f == s:
reg_exp_start += re.escape(f)
else:
reg_exp_start += ".*"
break
if reg_exp_start.endswith(".*"):
for f, s in zip(first_loc[::-1], second_loc[::-1]):
if f == s:
reg_exp_end = re.escape(f) + reg_exp_end
else:
break
return unquote(reg_exp_start + reg_exp_end)
def __init__(self, script_path, arguments, output):
global VERSION
program_banner = open(
FileUtils.buildPath(script_path, "lib", "controller",
"banner.txt")).read().format(**VERSION)
self.script_path = script_path
self.exit = False
self.arguments = arguments
self.output = output
self.savePath = self.script_path
self.doneDirs = []
self.recursive_level_max = self.arguments.recursive_level_max
if self.arguments.httpmethod.lower() not in ["get", "head", "post"]:
self.output.error("Inavlid http method!")
exit(1)
self.httpmethod = self.arguments.httpmethod.lower()
if self.arguments.saveHome:
savePath = self.getSavePath()
if not FileUtils.exists(savePath):
FileUtils.createDirectory(savePath)
if FileUtils.exists(savePath) and not FileUtils.isDir(savePath):
self.output.error(
'Cannot use {} because is a file. Should be a directory'.
format(savePath))
exit(1)
if not FileUtils.canWrite(savePath):
self.output.error(
'Directory {} is not writable'.format(savePath))
exit(1)
logs = FileUtils.buildPath(savePath, "logs")
if not FileUtils.exists(logs):
FileUtils.createDirectory(logs)
reports = FileUtils.buildPath(savePath, "reports")
if not FileUtils.exists(reports):
FileUtils.createDirectory(reports)
self.savePath = savePath
self.reportsPath = FileUtils.buildPath(self.savePath, "logs")
self.blacklists = self.getBlacklists()
self.blacklists = {}
self.fuzzer = None
self.excludeStatusCodes = self.arguments.excludeStatusCodes
self.excludeTexts = self.arguments.excludeTexts
self.excludeRegexps = self.arguments.excludeRegexps
self.recursive = self.arguments.recursive
self.suppressEmpty = self.arguments.suppressEmpty
self.directories = Queue()
self.excludeSubdirs = (arguments.excludeSubdirs
if arguments.excludeSubdirs is not None else [])
self.output.header(program_banner)
# self.dictionary = Dictionary(self.arguments.wordlist, self.arguments.extensions,
# self.arguments.lowercase, self.arguments.forceExtensions)
# self.printConfig()
self.errorLog = None
self.errorLogPath = None
self.errorLogLock = Lock()
self.batch = False
self.batchSession = None
self.setupErrorLogs()
self.output.newLine("\nError Log: {0}".format(self.errorLogPath))
if self.arguments.autoSave and len(self.arguments.urlList) > 1:
self.setupBatchReports()
self.output.newLine("\nAutoSave path: {0}".format(
self.batchDirectoryPath))
if self.arguments.useRandomAgents:
self.randomAgents = FileUtils.getLines(
FileUtils.buildPath(script_path, "db", "user-agents.txt"))
try:
for url in self.arguments.urlList:
try:
gc.collect()
self.reportManager = ReportManager()
self.currentUrl = url
self.output.target(self.currentUrl)
try:
# DNS A Record query
self.requester = Requester(
url,
script_path=self.script_path,
cookie=self.arguments.cookie,
useragent=self.arguments.useragent,
maxPool=self.arguments.threadsCount,
maxRetries=self.arguments.maxRetries,
delay=self.arguments.delay,
timeout=self.arguments.timeout,
ip=self.arguments.ip,
proxy=self.arguments.proxy,
redirect=self.arguments.redirect,
requestByHostname=self.arguments.requestByHostname,
httpmethod=self.httpmethod)
# 网站连通性测试
site_connection_test_resp = self.requester.request(
self.requester.basePath,
use_base_path=False,
allow_redirect=True,
fingerprint=True)
self.dictionary = Dictionary(self.requester.scan_list,
self.requester.directory,
self.requester.filename,
self.requester.extension)
# 404 page
if self.requester.url_type == URLType.normal_restful_dir:
path_404 = '{}/{}/'.format(
self.requester.basePath,
RandomUtils.randString(8))
path_404 = path_404.replace("//", "/")
elif self.requester.url_type == URLType.restful_file:
path_404 = self.requester.basePath.replace(
self.requester.filename,
RandomUtils.randString(
len(self.requester.filename) or 8))
elif self.requester.url_type == URLType.normal_file:
path_404 = self.requester.basePath.replace(
self.requester.filename,
RandomUtils.randString(
len(self.requester.filename) or 8))
path_404_quote = self.dictionary.quote(path_404)
response_404 = self.requester.request(
path_404_quote,
use_base_path=False,
allow_redirect=False)
# Waf 探测
waf_exist, waf_response = self.requester.waf_detect(
site_connection_test_resp.body,
url_quote=self.dictionary.quote)
except RequestException as e:
self.output.error(e.args[0]['message'])
raise SkipTargetInterrupt
if self.arguments.useRandomAgents:
self.requester.setRandomAgents(self.randomAgents)
for key, value in arguments.headers.items():
self.requester.setHeader(key, value)
# Initialize directories Queue with start Path
self.basePath = self.requester.basePath
if self.arguments.scanSubdirs is not None:
for subdir in self.arguments.scanSubdirs:
self.directories.put(subdir)
else:
self.directories.put('')
self.setupReports(self.requester)
matchCallbacks = [self.matchCallback]
notFoundCallbacks = [self.notFoundCallback]
errorCallbacks = [self.errorCallback, self.appendErrorLog]
self.fuzzer = Fuzzer(
self.requester,
self.dictionary,
waf_exist,
waf_response,
response_404,
testFailPath=self.arguments.testFailPath,
threads=self.arguments.threadsCount,
matchCallbacks=matchCallbacks,
notFoundCallbacks=notFoundCallbacks,
errorCallbacks=errorCallbacks)
try:
self.wait()
except RequestException as e:
self.output.error(
"Fatal error during site scanning: " +
e.args[0]['message'])
raise SkipTargetInterrupt
except SkipTargetInterrupt:
continue
finally:
self.reportManager.save()
except KeyboardInterrupt:
self.output.error('\nCanceled by the user')
exit(0)
finally:
if not self.errorLog.closed:
self.errorLog.close()
self.reportManager.close()
self.output.warning('\nTask Completed')
