def write_log(self, msg):
if not self.options["log_file"]:
return
line = time.strftime("[%y-%m-%d %H:%M:%S] ")
line += msg + NEW_LINE
FileUtils.write_lines(self.options["log_file"], line)
def log(file, tag, msg):
if not file:
return
line = time.strftime("[%y-%m-%d %H:%M:%S]")
line += f"[{tag.upper()}] {msg}"
line += NEW_LINE
FileUtils.write_lines(file, line)
def validate_dir(self, path):
if not FileUtils.exists(path):
self.validate_output_loc(FileUtils.parent(path))
if not FileUtils.is_dir(path):
self.output.error(
"{0} is a file, should be a directory".format(path))
exit(1)
if not FileUtils.can_write(path):
self.output.error("Directory {0} is not writable".format(path))
exit(1)
def setup(self, options):
self.options = options
self.pass_dirs = ['']
if options["raw_file"]:
self.options.update(
zip(["urls", "httpmethod", "headers", "data"],
parse_raw(options["raw_file"])))
else:
self.options["headers"] = {**DEFAULT_HEADERS, **options["headers"]}
if options["cookie"]:
self.options["headers"]["Cookie"] = options["cookie"]
if options["useragent"]:
self.options["headers"]["User-Agent"] = options["useragent"]
self.random_agents = None
if options["use_random_agents"]:
self.random_agents = FileUtils.get_lines(
FileUtils.build_path(SCRIPT_PATH, "db", "user-agents.txt"))
self.targets.queue = deque(options["urls"])
self.blacklists = Dictionary.generate_blacklists(options["extensions"])
self.dictionary = Dictionary(
paths=options["wordlist"],
extensions=options["extensions"],
suffixes=options["suffixes"],
prefixes=options["prefixes"],
lowercase=options["lowercase"],
uppercase=options["uppercase"],
capitalization=options["capitalization"],
force_extensions=options["force_extensions"],
exclude_extensions=options["exclude_extensions"],
no_extension=options["no_extension"],
only_selected=options["only_selected"])
self.current_job = 0
self.batch = False
self.batch_session = None
self.exit = None
self.start_time = time.time()
self.jobs_count = self.targets.qsize() * (len(
options["scan_subdirs"]) if options["scan_subdirs"] else 1)
if options["autosave_report"] or options["output_file"]:
if options["autosave_report"]:
self.report_path = options[
"output_location"] or FileUtils.build_path(
SCRIPT_PATH, "reports")
self.create_dir(self.report_path)
if options["log_file"]:
options["log_file"] = FileUtils.get_abs_path(options["log_file"])
self.create_dir(FileUtils.parent(options["log_file"]))
def validate_path(self, path):
if not FileUtils.exists(path):
self.output.error("{0} does not exist".format(path))
exit(1)
if FileUtils.exists(path) and not FileUtils.is_dir(path):
self.output.error("{0} is a file, should be a directory".format(path))
exit(1)
if not FileUtils.can_write(path):
self.output.error("Directory {0} is not writable".format(path))
exit(1)
return True
def setup_batch_reports(self):
"""Create batch report folder"""
self.batch = True
current_time = time.strftime("%y-%m-%d_%H-%M-%S")
batch_session = f"BATCH-{current_time}"
batch_directory_path = FileUtils.build_path(self.report_path, batch_session)
try:
FileUtils.create_dir(batch_directory_path)
except Exception:
self.output.error(f"Couldn't create batch folder at {batch_directory_path}")
exit(1)
return batch_directory_path
def generate_blacklists(extensions):
blacklists = {}
for status in [400, 403, 500]:
blacklist_file_name = FileUtils.build_path(SCRIPT_PATH, "db")
blacklist_file_name = FileUtils.build_path(
blacklist_file_name, "{}_blacklist.txt".format(status))
if not FileUtils.can_read(blacklist_file_name):
# Skip if cannot read file
continue
blacklists[status] = list(
Dictionary([blacklist_file_name], extensions))
return blacklists
def setup_batch_reports(self):
self.batch = True
if not self.options["output_file"]:
self.batch_session = "BATCH-{0}".format(
time.strftime("%y-%m-%d_%H-%M-%S"))
self.batch_directory_path = FileUtils.build_path(
self.report_path, self.batch_session)
if not FileUtils.exists(self.batch_directory_path):
FileUtils.create_directory(self.batch_directory_path)
if not FileUtils.exists(self.batch_directory_path):
self.output.error(
"Couldn't create batch folder at {}".format(
self.batch_directory_path))
exit(1)
def __init__(self, options, output):
self.targets = Queue()
self.directories = Queue()
self.threads_lock = threading.Lock()
self.report_manager = EmptyReportManager()
self.report = EmptyReport()
self.output = output
if options["session_file"]:
self._import(FileUtils.read(options["session_file"]))
self.from_export = True
else:
self.setup(options)
self.from_export = False
self.output.header(BANNER)
self.output.config(
', '.join(self.options["extensions"]),
', '.join(self.options["prefixes"]),
', '.join(self.options["suffixes"]),
str(self.options["threads_count"]),
str(len(self.dictionary)),
str(self.options["httpmethod"]),
)
self.setup_reports()
self.output.log_file(options["log_file"])
try:
self.run()
except KeyboardInterrupt:
self.close("Canceled by the user")
def setup_reports(self):
if self.output_file:
output_file = FileUtils.get_abs_path(self.output_file)
self.output.output_file(output_file)
else:
if len(self.url_list) > 1:
self.setup_batch_reports()
filename = "BATCH"
filename += self.get_output_extension()
directory_path = self.batch_directory_path
else:
parsed = urlparse(self.url_list[0])
filename = (
"{}_".format(parsed.path)
)
filename += time.strftime("%y-%m-%d_%H-%M-%S")
filename += self.get_output_extension()
directory_path = FileUtils.build_path(
self.report_path, clean_filename(parsed.netloc)
)
filename = clean_filename(filename)
output_file = FileUtils.build_path(directory_path, filename)
if FileUtils.exists(output_file):
i = 2
while FileUtils.exists(output_file + "_" + str(i)):
i += 1
output_file += "_" + str(i)
if not FileUtils.exists(directory_path):
FileUtils.create_directory(directory_path)
if not FileUtils.exists(directory_path):
self.output.error(
"Couldn't create the reports folder at {}".format(directory_path)
)
sys.exit(1)
self.output.output_file(output_file)
if self.output_file and self.output_format:
self.report_manager = ReportManager(self.output_format, self.output_file)
elif self.output_format:
self.report_manager = ReportManager(self.output_format, output_file)
else:
self.report_manager = ReportManager("plain", output_file)
def _export(self, session_file):
self.targets.queue.insert(0, self.url)
self.directories.queue.insert(0, self.current_directory)
# Queue() objects, convert them to list
for item in ("targets", "directories"):
self.__dict__[item] = list(self.__dict__[item].queue)
self.dictionary, self.dictionary_index = self.dictionary.export()
self.last_output = self.output.export()
self.current_job -= 1
data = {
k: v
for k, v in self.__dict__.items()
if k not in EXCLUDED_EXPORT_VARIABLES
}
FileUtils.write_lines(session_file, str(data), overwrite=True)
def setup_error_logs(self):
file_name = "errors-{0}.log".format(time.strftime("%y-%m-%d_%H-%M-%S"))
self.error_log_path = FileUtils.build_path(self.logs_path, file_name)
try:
self.error_log = open(self.error_log_path, "w")
except PermissionError:
self.output.error(
"Couldn't create the error log. Try running again with highest permission"
)
sys.exit(1)
def setup_reports(self):
if self.options["output_file"]:
output_file = FileUtils.get_abs_path(self.options["output_file"])
self.output.output_file(output_file)
else:
if self.targets.qsize() > 1:
self.setup_batch_reports()
filename = "BATCH"
filename += self.get_output_extension()
directory_path = self.batch_directory_path
else:
parsed = urlparse(self.targets.queue[0])
filename = ("{}_".format(parsed.path))
filename += time.strftime("%y-%m-%d_%H-%M-%S")
filename += self.get_output_extension()
directory_path = FileUtils.build_path(
self.report_path, get_valid_filename(parsed.netloc))
filename = get_valid_filename(filename)
output_file = FileUtils.build_path(directory_path, filename)
if FileUtils.exists(output_file):
i = 2
while FileUtils.exists(output_file + "_" + str(i)):
i += 1
output_file += "_" + str(i)
if not FileUtils.exists(directory_path):
FileUtils.create_directory(directory_path)
if not FileUtils.exists(directory_path):
self.output.error(
"Couldn't create the reports folder at {}".format(
directory_path))
exit(1)
self.output.output_file(output_file)
if self.options["output_format"]:
self.report_manager = ReportManager(
self.options["output_format"], self.options["output_file"]
or output_file)
else:
self.report_manager = ReportManager("plain", output_file)
def generate_blacklists(extensions, script_path):
reext = re.compile(r"\%ext\%", re.IGNORECASE).sub
blacklists = {}
for status in [400, 403, 500]:
blacklist_file_name = FileUtils.build_path(script_path, "db")
blacklist_file_name = FileUtils.build_path(
blacklist_file_name, "{}_blacklist.txt".format(status)
)
if not FileUtils.can_read(blacklist_file_name):
# Skip if cannot read file
continue
blacklists[status] = []
for line in FileUtils.get_lines(blacklist_file_name):
# Skip comments
if line.lstrip().startswith("#"):
continue
if line.startswith("/"):
line = line[1:]
# Classic dirsearch blacklist processing (with %EXT% keyword)
if "%ext%" in line.lower():
for extension in extensions:
entry = reext.sub(extension, line)
blacklists[status].append(entry)
# Forced extensions is not used here because -r is only used for wordlist,
# applying in blacklist may create false negatives
else:
blacklists[status].append(line)
return blacklists
def create_dir(self, path):
if path == '/':
return
if not FileUtils.exists(path):
self.create_dir(FileUtils.parent(path))
if not FileUtils.is_dir(path):
self.output.error(
"{0} is a file, should be a directory".format(path))
exit(1)
if not FileUtils.can_write(path):
self.output.error("Directory {0} is not writable".format(path))
exit(1)
FileUtils.create_directory(path)
def setup_reports(self):
"""Create report file"""
output_file = None
if self.options.output_file:
output_file = FileUtils.get_abs_path(self.options.output_file)
elif self.options.autosave_report:
if len(self.targets) > 1:
directory_path = self.setup_batch_reports()
filename = "BATCH" + self.get_output_extension()
else:
parsed = urlparse(self.options.urls[0])
filename = get_valid_filename(f"{parsed.path}_")
filename += time.strftime("%y-%m-%d_%H-%M-%S")
filename += self.get_output_extension()
directory_path = FileUtils.build_path(
self.report_path, get_valid_filename(parsed.netloc)
)
output_file = FileUtils.build_path(directory_path, filename)
if FileUtils.exists(output_file):
i = 2
while FileUtils.exists(f"{output_file}_{i}"):
i += 1
output_file += f"_{i}"
try:
FileUtils.create_dir(directory_path)
except Exception:
self.output.error(
f"Couldn't create the reports folder at {directory_path}"
)
exit(1)
self.report_manager = ReportManager(self.options.output_format, output_file)
if output_file:
self.output.output_file(output_file)
def setup(self, options, output):
self.options, self.output = options, output
if self.options.raw_file:
self.options.update(
zip(
["urls", "httpmethod", "headers", "data"],
parse_raw(self.options.raw_file),
)
)
else:
self.options.headers = {**DEFAULT_HEADERS, **self.options.headers}
if self.options.cookie:
self.options.headers["Cookie"] = self.options.cookie
if self.options.useragent:
self.options.headers["User-Agent"] = self.options.useragent
self.random_agents = None
if self.options.use_random_agents:
self.random_agents = FileUtils.get_lines(
FileUtils.build_path(SCRIPT_PATH, "db", "user-agents.txt")
)
self.requester = Requester(
max_pool=self.options.threads_count,
max_retries=self.options.max_retries,
timeout=self.options.timeout,
ip=self.options.ip,
proxy=self.options.proxy,
follow_redirects=self.options.follow_redirects,
httpmethod=self.options.httpmethod,
headers=self.options.headers,
data=self.options.data,
scheme=self.options.scheme,
random_agents=self.random_agents,
)
self.dictionary = Dictionary(
paths=self.options.wordlist,
extensions=self.options.extensions,
suffixes=self.options.suffixes,
prefixes=self.options.prefixes,
lowercase=self.options.lowercase,
uppercase=self.options.uppercase,
capitalization=self.options.capitalization,
force_extensions=self.options.force_extensions,
exclude_extensions=self.options.exclude_extensions,
no_extension=self.options.no_extension,
only_selected=self.options.only_selected,
)
self.blacklists = Dictionary.generate_blacklists(self.options.extensions)
self.targets = options.urls
self.start_time = time.time()
self.passed_urls = set()
self.directories = []
self.report = None
self.batch = False
self.current_job = 0
self.jobs_count = 0
self.errors = 0
self.consecutive_errors = 0
if self.options.auth:
self.requester.set_auth(self.options.auth_type, self.options.auth)
if self.options.proxy_auth:
self.requester.set_proxy_auth(self.options.proxy_auth)
if self.options.log_file:
self.options.log_file = FileUtils.get_abs_path(self.options.log_file)
try:
FileUtils.create_dir(FileUtils.parent(self.options.log_file))
if not FileUtils.can_write(self.options.log_file):
raise Exception
except Exception:
self.output.error(
f"Couldn't create log file at {self.options.log_file}"
)
exit(1)
if self.options.autosave_report:
self.report_path = self.options.output_path or FileUtils.build_path(
SCRIPT_PATH, "reports"
)
try:
FileUtils.create_dir(self.report_path)
if not FileUtils.can_write(self.report_path):
raise Exception
except Exception:
self.output.error(
f"Couldn't create report folder at {self.report_path}"
)
exit(1)
self.output.header(BANNER)
self.output.config(
", ".join(self.options["extensions"]),
", ".join(self.options["prefixes"]),
", ".join(self.options["suffixes"]),
str(self.options["threads_count"]),
str(len(self.dictionary)),
str(self.options["httpmethod"]),
)
self.setup_reports()
if self.options.log_file:
self.output.log_file(self.options.log_file)
def parse_arguments():
usage = "Usage: %prog [-u|--url] target [-e|--extensions] extensions [options]"
parser = OptionParser(usage, version="dirsearch v{}".format(VERSION))
# Mandatory arguments
mandatory = OptionGroup(parser, "Mandatory")
mandatory.add_option("-u",
"--url",
action="append",
dest="urls",
metavar="URL",
help="Target URL(s), support multiple flags")
mandatory.add_option("-l",
"--url-list",
action="store",
dest="url_list",
metavar="FILE",
help="URL list file")
mandatory.add_option("--stdin",
action="store_true",
dest="stdin_urls",
help="Read URL(s) from STDIN")
mandatory.add_option("--cidr",
action="store",
dest="cidr",
help="Target CIDR")
mandatory.add_option(
"--raw",
action="store",
dest="raw_file",
metavar="FILE",
help=
"Load raw HTTP request from file (use `--scheme` flag to set the scheme)"
)
mandatory.add_option("-s",
"--session",
action="store",
dest="session_file",
help="Session file")
mandatory.add_option(
"-e",
"--extensions",
action="store",
dest="extensions",
help="Extension list separated by commas (Example: php,asp)")
mandatory.add_option(
"-X",
"--exclude-extensions",
action="store",
dest="exclude_extensions",
metavar="EXTENSIONS",
help="Exclude extension list separated by commas (Example: asp,jsp)")
mandatory.add_option(
"-f",
"--force-extensions",
action="store_true",
dest="force_extensions",
help=
"Add extensions to every wordlist entry. By default dirsearch only replaces the %EXT% keyword with extensions"
)
mandatory.add_option(
"--config",
action="store",
dest="config",
default=FileUtils.build_path(SCRIPT_PATH, "default.conf"),
metavar="FILE",
help=
"Full path to config file, see 'default.conf' for example (Default: default.conf)"
)
# Dictionary Settings
dictionary = OptionGroup(parser, "Dictionary Settings")
dictionary.add_option("-w",
"--wordlists",
action="store",
dest="wordlist",
help="Customize wordlists (separated by commas)")
dictionary.add_option(
"--prefixes",
action="store",
dest="prefixes",
help="Add custom prefixes to all wordlist entries (separated by commas)"
)
dictionary.add_option(
"--suffixes",
action="store",
dest="suffixes",
help=
"Add custom suffixes to all wordlist entries, ignore directories (separated by commas)"
)
dictionary.add_option(
"--only-selected",
action="store_true",
dest="only_selected",
help=
"Remove paths have different extensions from selected ones via `-e` (keep entries don't have extensions)"
)
dictionary.add_option(
"--remove-extensions",
action="store_true",
dest="no_extension",
help="Remove extensions in all paths (Example: admin.php -> admin)")
dictionary.add_option("-U",
"--uppercase",
action="store_true",
dest="uppercase",
help="Uppercase wordlist")
dictionary.add_option("-L",
"--lowercase",
action="store_true",
dest="lowercase",
help="Lowercase wordlist")
dictionary.add_option("-C",
"--capital",
action="store_true",
dest="capitalization",
help="Capital wordlist")
# Optional Settings
general = OptionGroup(parser, "General Settings")
general.add_option("-t",
"--threads",
action="store",
type="int",
dest="threads_count",
metavar="THREADS",
help="Number of threads")
general.add_option("-r",
"--recursive",
action="store_true",
dest="recursive",
help="Brute-force recursively")
general.add_option(
"--deep-recursive",
action="store_true",
dest="deep_recursive",
help=
"Perform recursive scan on every directory depth (Example: api/users -> api/)"
)
general.add_option(
"--force-recursive",
action="store_true",
dest="force_recursive",
help=
"Do recursive brute-force for every found path, not only directories")
general.add_option("-R",
"--max-recursion-depth",
action="store",
type="int",
dest="recursion_depth",
metavar="DEPTH",
help="Maximum recursion depth")
general.add_option(
"--recursion-status",
action="store",
dest="recursion_status_codes",
metavar="CODES",
help=
"Valid status codes to perform recursive scan, support ranges (separated by commas)"
)
general.add_option(
"--subdirs",
action="store",
dest="scan_subdirs",
metavar="SUBDIRS",
help="Scan sub-directories of the given URL[s] (separated by commas)")
general.add_option(
"--exclude-subdirs",
action="store",
dest="exclude_subdirs",
metavar="SUBDIRS",
help=
"Exclude the following subdirectories during recursive scan (separated by commas)"
)
general.add_option(
"-i",
"--include-status",
action="store",
dest="include_status_codes",
metavar="CODES",
help=
"Include status codes, separated by commas, support ranges (Example: 200,300-399)"
)
general.add_option(
"-x",
"--exclude-status",
action="store",
dest="exclude_status_codes",
metavar="CODES",
help=
"Exclude status codes, separated by commas, support ranges (Example: 301,500-599)"
)
general.add_option(
"--exclude-sizes",
action="store",
dest="exclude_sizes",
metavar="SIZES",
help="Exclude responses by sizes, separated by commas (Example: 0B,4KB)"
)
general.add_option(
"--exclude-texts",
action="store",
dest="exclude_texts",
metavar="TEXTS",
help=
"Exclude responses by texts, separated by commas (Example: 'Not found', 'Error')"
)
general.add_option("--exclude-regex",
action="store",
dest="exclude_regex",
metavar="REGEX",
help="Exclude responses by regex (Example: '^Error$')")
general.add_option(
"--exclude-redirect",
action="store",
dest="exclude_redirect",
metavar="STRING",
help=
"Exclude responses if this regex (or text) matches redirect URL (Example: '/index.html')",
)
general.add_option(
"--exclude-response",
action="store",
dest="exclude_response",
metavar="PATH",
help=
"Exclude responses similar to response of this page, path as input (Example: 404.html)"
)
general.add_option(
"--skip-on-status",
action="store",
dest="skip_on_status",
metavar="CODES",
help=
"Skip target whenever hit one of these status codes, separated by commas, support ranges"
)
general.add_option("--min-response-size",
action="store",
type="int",
dest="minimum_response_size",
help="Minimum response length",
metavar="LENGTH",
default=0)
general.add_option("--max-response-size",
action="store",
type="int",
dest="maximum_response_size",
help="Maximum response length",
metavar="LENGTH",
default=0)
general.add_option("--redirects-history",
action="store_true",
dest="redirects_history",
help="Show redirects history")
general.add_option("--max-time",
action="store",
type="int",
dest="maxtime",
metavar="SECONDS",
help="Maximum runtime for the scan")
general.add_option(
"--full-url",
action="store_true",
dest="full_url",
help="Full URLs in the output (enabled automatically in quiet mode)")
general.add_option("--no-color",
action="store_false",
dest="color",
help="No colored output")
general.add_option("-q",
"--quiet-mode",
action="store_true",
dest="quiet",
help="Quiet mode")
# Request Settings
request = OptionGroup(parser, "Request Settings")
request.add_option("-m",
"--http-method",
action="store",
dest="httpmethod",
metavar="METHOD",
help="HTTP method (default: GET)")
request.add_option("-d",
"--data",
action="store",
dest="data",
help="HTTP request data")
request.add_option("-H",
"--header",
action="append",
dest="headers",
help="HTTP request header, support multiple flags")
request.add_option("--header-list",
dest="header_list",
metavar="FILE",
help="File contains HTTP request headers")
request.add_option("-F",
"--follow-redirects",
action="store_true",
dest="follow_redirects",
help="Follow HTTP redirects")
request.add_option("--random-agent",
action="store_true",
dest="use_random_agents",
help="Choose a random User-Agent for each request")
request.add_option("--auth-type",
action="store",
dest="auth_type",
metavar="TYPE",
help="Authentication type ({})".format(
", ".join(AUTHENTICATION_TYPES)))
request.add_option(
"--auth",
action="store",
dest="auth",
metavar="CREDENTIAL",
help="Authentication credential ([user]:[password] or bearer token)")
request.add_option("--user-agent", action="store", dest="useragent")
request.add_option("--cookie", action="store", dest="cookie")
# Connection Settings
connection = OptionGroup(parser, "Connection Settings")
connection.add_option("--timeout",
action="store",
type="float",
dest="timeout",
help="Connection timeout")
connection.add_option("--delay",
action="store",
type="float",
dest="delay",
help="Delay between requests")
connection.add_option(
"--proxy",
action="store",
dest="proxy",
metavar="PROXY",
help=
"Proxy URL, support HTTP and SOCKS proxies (Example: localhost:8080, socks5://localhost:8088)"
)
connection.add_option("--proxy-list",
action="store",
type="string",
dest="proxylist",
help="File contains proxy servers",
metavar="FILE")
connection.add_option("--replay-proxy",
action="store",
dest="replay_proxy",
metavar="PROXY",
help="Proxy to replay with found paths")
connection.add_option(
"--scheme",
action="store",
dest="scheme",
metavar="SCHEME",
help=
"Scheme for raw request or if there is no scheme in the URL (Default: auto-detect)"
)
connection.add_option("--max-rate",
action="store",
type="int",
dest="maxrate",
metavar="RATE",
help="Max requests per second")
connection.add_option("--retries",
action="store",
type="int",
dest="max_retries",
metavar="RETRIES",
help="Number of retries for failed requests")
connection.add_option(
"-b",
"--request-by-hostname",
action="store_true",
dest="request_by_hostname",
help=
"By default dirsearch requests by IP for speed. This will force dirsearch to request by hostname"
)
connection.add_option("--ip",
action="store",
dest="ip",
help="Server IP address")
connection.add_option("--exit-on-error",
action="store_true",
dest="exit_on_error",
help="Exit whenever an error occurs")
# Output Settings
output = OptionGroup(parser, "Output")
output.add_option("-o",
"--output",
action="store",
dest="output_file",
metavar="FILE",
help="Output file")
output.add_option(
"--format",
action="store",
dest="output_format",
metavar="FORMAT",
help=
"Report format (Available: simple, plain, json, xml, md, csv, html, sqlite)"
)
output.add_option("--log",
action="store",
dest="log_file",
metavar="FILE",
help="Log file")
parser.add_option_group(mandatory)
parser.add_option_group(dictionary)
parser.add_option_group(general)
parser.add_option_group(request)
parser.add_option_group(connection)
parser.add_option_group(output)
options, arguments = parser.parse_args()
return options
def parse_config(self):
config = ConfigParser()
config_path = FileUtils.build_path(self.script_path, "default.conf")
config.read(config_path)
# Mandatory
self.default_extensions = config.safe_get("mandatory",
"default-extensions", str())
self.exclude_extensions = config.safe_get("mandatory",
"exclude-extensions", None)
self.force_extensions = config.safe_getboolean("mandatory",
"force-extensions",
False)
# General
self.threads_count = config.safe_getint("general", "threads", 30,
list(range(1, 300)))
self.include_status_codes = config.safe_get("general",
"include-status", None)
self.exclude_status_codes = config.safe_get("general",
"exclude-status", None)
self.exclude_sizes = config.safe_get("general", "exclude-sizes", None)
self.exclude_texts = config.safe_get("general", "exclude-texts", None)
self.exclude_regexps = config.safe_get("general", "exclude-regexps",
None)
self.exclude_redirects = config.safe_get("general",
"exclude-redirects", None)
self.exclude_content = config.safe_get("general", "exclude-content",
"")
self.recursive = config.safe_getboolean("general", "recursive", False)
self.deep_recursive = config.safe_getboolean("general",
"deep-recursive", False)
self.force_recursive = config.safe_getboolean("general",
"force-recursive", False)
self.recursion_depth = config.safe_getint("general", "recursion-depth",
0)
self.recursion_status_codes = config.safe_get("general",
"recursion-status", None)
self.exclude_subdirs = config.safe_get("general", "exclude-subdirs",
None)
self.skip_on_status = config.safe_get("general", "skip-on-status",
None)
self.maxtime = config.safe_getint("general", "max-time", 0)
self.full_url = config.safe_getboolean("general", "full-url", False)
self.color = config.safe_getboolean("general", "color", True)
self.quiet = config.safe_getboolean("general", "quiet-mode", False)
# Reports
self.output_location = config.safe_get("reports",
"report-output-folder", None)
self.autosave_report = config.safe_getboolean("reports",
"autosave-report", False)
self.logs_location = config.safe_get("reports", "logs-location", None)
self.output_format = config.safe_get(
"reports", "report-format", "plain",
["plain", "simple", "json", "xml", "md", "csv", "html"])
# Dictionary
self.wordlist = config.safe_get(
"dictionary",
"wordlist",
FileUtils.build_path(self.script_path, "db", "dicc.txt"),
)
self.prefixes = config.safe_get("dictionary", "prefixes", None)
self.suffixes = config.safe_get("dictionary", "suffixes", None)
self.lowercase = config.safe_getboolean("dictionary", "lowercase",
False)
self.uppercase = config.safe_getboolean("dictionary", "uppercase",
False)
self.capitalization = config.safe_getboolean("dictionary",
"capitalization", False)
# Request
self.httpmethod = config.safe_get("request", "httpmethod", "get")
self.header_list = config.safe_get("request", "headers-file", None)
self.redirect = config.safe_getboolean("request", "follow-redirects",
False)
self.use_random_agents = config.safe_get("request",
"random-user-agents", False)
self.useragent = config.safe_get("request", "user-agent", "")
self.cookie = config.safe_get("request", "cookie", "")
# Connection
self.delay = config.safe_getfloat("connection", "delay", 0)
self.timeout = config.safe_getint("connection", "timeout", 10)
self.max_retries = config.safe_getint("connection", "retries", 2)
self.maxrate = config.safe_getint("connection", "max-rate", 0)
self.proxy = config.safe_get("connection", "proxy", None)
self.proxylist = config.safe_get("connection", "proxy-list", None)
self.scheme = config.safe_get("connection", "scheme", "http",
["http", "https"])
self.replay_proxy = config.safe_get("connection", "replay-proxy", None)
self.request_by_hostname = config.safe_getboolean(
"connection", "request-by-hostname", False)
self.exit_on_error = config.safe_getboolean("connection",
"exit-on-error", False)
import os
import sys
import string
from lib.utils.file import FileUtils
# Version format: <major version>.<minor version>.<revision>[.<month>]
VERSION = "0.4.2.4"
BANNER = f"""
_|. _ _ _ _ _ _|_ v{VERSION}
(_||| _) (/_(_|| (_| )
"""
SCRIPT_PATH = FileUtils.parent(__file__, 3)
IS_WINDOWS = sys.platform in ("win32", "msys")
DEFAULT_ENCODING = "utf-8"
NEW_LINE = os.linesep
INVALID_CHARS_FOR_WINDOWS_FILENAME = ('"', "*", "<", ">", "?", "\\", "|", "/", ":")
INVALID_FILENAME_CHAR_REPLACEMENT = "_"
OUTPUT_FORMATS = ("simple", "plain", "json", "xml", "md", "csv", "html", "sqlite")
COMMON_EXTENSIONS = ("php", "jsp", "asp", "aspx", "do", "action", "cgi", "html", "htm", "js", "json", "tar.gz", "bak")
def get_dependencies():
try:
return FileUtils.get_lines(REQUIREMENTS_FILE)
except FileNotFoundError:
print("Can't find requirements.txt")
exit(1)
def generate(self):
"""
Dictionary.generate() behaviour
Classic dirsearch wordlist:
1. If %EXT% keyword is present, append one with each extension REPLACED.
2. If the special word is no present, append line unmodified.
Forced extensions wordlist (NEW):
This type of wordlist processing is a mix between classic processing
and DirBuster processing.
1. If %EXT% keyword is present in the line, immediately process as "classic dirsearch" (1).
2. If the line does not include the special word AND is NOT terminated by a slash,
append one with each extension APPENDED (line.ext) and ONLYE ONE with a slash.
3. If the line does not include the special word and IS ALREADY terminated by slash,
append line unmodified.
"""
reext = re.compile(EXTENSION_TAG, re.IGNORECASE)
result = []
# Enable to use multiple dictionaries at once
for dict_file in self._dictionary_files:
for line in FileUtils.get_lines(dict_file):
if line.startswith("/"):
line = line[1:]
if self.no_extension:
line = line.split(".")[0]
# Skip comments
if line.lstrip().startswith("#"):
continue
# Skip if the path contains excluded extensions
if any("." + extension in line
for extension in self.exclude_extensions):
continue
# Classic dirsearch wordlist processing (with %EXT% keyword)
if EXTENSION_TAG in line.lower():
for extension in self.extensions:
newline = reext.sub(extension, line)
result.append(newline)
# If "forced extensions" is used and the path is not a directory (terminated by /) or has
# had an extension already, append extensions to the path
elif (self.force_extensions and not line.endswith("/")
and not re.search(EXTENSION_REGEX, line)):
for extension in self.extensions:
result.append(line + f".{extension}")
result.append(line)
result.append(line + "/")
# Append line unmodified.
elif not self.only_selected or any(
line.endswith(f".{extension}")
for extension in self.extensions):
result.append(line)
# Re-add dictionary with prefixes
result.extend(pref + path for path in result for pref in self.prefixes
if not path.startswith(pref))
# Re-add dictionary with suffixes
result.extend(path + suff for path in result for suff in self.suffixes
if not path.endswith(("/", suff)))
if self.lowercase:
self._entries = tuple(entry.lower() for entry in uniq(result))
elif self.uppercase:
self._entries = tuple(entry.upper() for entry in uniq(result))
elif self.capitalization:
self._entries = tuple(entry.capitalize() for entry in uniq(result))
else:
self._entries = tuple(uniq(result))
del result
def __init__(self, options, output):
self.directories = Queue()
self.output = output
self.options = options
self.pass_dirs = ["/"]
if options.raw_file:
raw = Raw(options.raw_file)
self.url_list = [raw.url]
self.httpmethod = raw.method
self.data = raw.body
self.headers = raw.headers
else:
self.url_list = options.url_list
self.httpmethod = options.httpmethod
self.data = options.httpmethod
self.headers = {**DEFAULT_HEADERS, **options.headers}
if options.cookie:
self.headers["Cookie"] = options.cookie
if options.useragent:
self.headers["User-Agent"] = options.useragent
self.random_agents = None
if options.use_random_agents:
self.random_agents = FileUtils.get_lines(
FileUtils.build_path(SCRIPT_PATH, "db", "user-agents.txt"))
self.blacklists = Dictionary.generate_blacklists(options.extensions)
self.dictionary = Dictionary(
paths=options.wordlist,
extensions=options.extensions,
suffixes=options.suffixes,
prefixes=options.prefixes,
lowercase=options.lowercase,
uppercase=options.uppercase,
capitalization=options.capitalization,
force_extensions=options.force_extensions,
exclude_extensions=options.exclude_extensions,
no_extension=options.no_extension,
only_selected=options.only_selected)
self.jobs_count = len(self.url_list) * (len(options.scan_subdirs)
if options.scan_subdirs else 1)
self.current_job = 0
self.batch = False
self.batch_session = None
self.exit = None
self.threads_lock = threading.Lock()
self.report_manager = EmptyReportManager()
self.report = EmptyReport()
self.start_time = time.time()
self.output.header(BANNER)
self.print_config()
if options.autosave_report or options.output_file:
if options.autosave_report:
self.report_path = options.output_location or FileUtils.build_path(
SCRIPT_PATH, "reports")
self.validate_dir(self.report_path)
self.setup_reports()
if options.log_file:
self.validate_dir(FileUtils.parent(options.log_file))
FileUtils.create_directory(FileUtils.parent(options.log_file))
self.output.log_file(FileUtils.get_abs_path(options.log_file))
try:
for url in self.url_list:
try:
gc.collect()
try:
self.requester = Requester(
url if url.endswith("/") else url + "/",
max_pool=options.threads_count,
max_retries=options.max_retries,
timeout=options.timeout,
ip=options.ip,
proxy=options.proxy,
proxylist=options.proxylist,
redirect=options.follow_redirects,
request_by_hostname=options.request_by_hostname,
httpmethod=self.httpmethod,
data=self.data,
scheme=options.scheme,
random_agents=self.random_agents,
)
self.output.set_target(self.requester.base_url +
self.requester.base_path)
self.requester.setup()
for key, value in self.headers.items():
self.requester.set_header(key, value)
if options.auth:
self.requester.set_auth(options.auth_type,
options.auth)
# Test request to check if server is up
self.requester.request("")
self.write_log("Test request sent for: {}".format(
self.requester.base_url))
if options.autosave_report or options.output_file:
self.report = Report(self.requester.host,
self.requester.port,
self.requester.scheme,
self.requester.base_path)
except RequestException as e:
self.output.error(e.args[0])
raise SkipTargetInterrupt
self.skip = None
if not options.scan_subdirs:
self.directories.put("")
for subdir in options.scan_subdirs:
self.directories.put(subdir)
self.pass_dirs.append(subdir)
match_callbacks = [self.match_callback, self.append_log]
not_found_callbacks = [
self.not_found_callback, self.append_log
]
error_callbacks = [
self.error_callback, self.append_error_log
]
self.fuzzer = Fuzzer(
self.requester,
self.dictionary,
suffixes=options.suffixes,
prefixes=options.prefixes,
exclude_response=options.exclude_response,
threads=options.threads_count,
delay=options.delay,
maxrate=options.maxrate,
match_callbacks=match_callbacks,
not_found_callbacks=not_found_callbacks,
error_callbacks=error_callbacks,
)
try:
self.prepare()
except RequestException as e:
self.output.error(e.args[0])
raise SkipTargetInterrupt
except SkipTargetInterrupt:
self.report.completed = True
continue
except KeyboardInterrupt:
self.close("Canceled by the user")
self.output.warning("\nTask Completed")
def parse_config(opt):
config = ConfigParser()
config.read(opt.config)
# Mandatory
opt.extensions = opt.extensions or config.safe_get("mandatory",
"default-extensions")
opt.exclude_extensions = opt.exclude_extensions or config.safe_get(
"mandatory", "exclude-extensions")
opt.force_extensions = opt.force_extensions or config.safe_getboolean(
"mandatory", "force-extensions")
# General
opt.threads_count = opt.threads_count or config.safe_getint(
"general", "threads", 25)
opt.include_status_codes = opt.include_status_codes or config.safe_get(
"general", "include-status")
opt.exclude_status_codes = opt.exclude_status_codes or config.safe_get(
"general", "exclude-status")
opt.exclude_sizes = opt.exclude_sizes or config.safe_get(
"general", "exclude-sizes")
opt.exclude_texts = opt.exclude_texts or config.safe_get(
"general", "exclude-texts")
opt.exclude_regex = opt.exclude_regex or config.safe_get(
"general", "exclude-regex")
opt.exclude_redirect = opt.exclude_redirect or config.safe_get(
"general", "exclude-redirect")
opt.exclude_response = opt.exclude_response or config.safe_get(
"general", "exclude-response")
opt.recursive = opt.recursive or config.safe_getboolean(
"general", "recursive")
opt.deep_recursive = opt.deep_recursive or config.safe_getboolean(
"general", "deep-recursive")
opt.force_recursive = opt.force_recursive or config.safe_getboolean(
"general", "force-recursive")
opt.recursion_depth = opt.recursion_depth or config.safe_getint(
"general", "max-recursion-depth")
opt.recursion_status_codes = opt.recursion_status_codes or config.safe_get(
"general", "recursion-status", "100-999")
opt.scan_subdirs = opt.scan_subdirs or config.safe_get(
"general", "subdirs")
opt.exclude_subdirs = opt.exclude_subdirs or config.safe_get(
"general", "exclude-subdirs")
opt.skip_on_status = opt.skip_on_status or config.safe_get(
"general", "skip-on-status")
opt.maxtime = opt.maxtime or config.safe_getint("general", "max-time")
opt.full_url = opt.full_url or config.safe_getboolean(
"general", "full-url")
opt.color = opt.color or config.safe_getboolean("general", "color", True)
opt.quiet = opt.quiet or config.safe_getboolean("general", "quiet-mode")
opt.redirects_history = opt.redirects_history or config.safe_getboolean(
"general", "show-redirects-history")
# Dictionary
opt.wordlist = opt.wordlist or config.safe_get(
"dictionary",
"wordlist",
FileUtils.build_path(SCRIPT_PATH, "db", "dicc.txt"),
)
opt.prefixes = opt.prefixes or config.safe_get(
"dictionary",
"prefixes",
)
opt.suffixes = opt.suffixes or config.safe_get("dictionary", "suffixes")
opt.lowercase = opt.lowercase or config.safe_getboolean(
"dictionary", "lowercase")
opt.uppercase = opt.uppercase or config.safe_getboolean(
"dictionary", "uppercase")
opt.capitalization = opt.capitalization or config.safe_getboolean(
"dictionary", "capitalization")
# Request
opt.httpmethod = opt.httpmethod or config.safe_get("request", "httpmethod",
"get")
opt.header_file = opt.header_file or config.safe_get(
"request", "headers-file")
opt.follow_redirects = opt.follow_redirects or config.safe_getboolean(
"request", "follow-redirects")
opt.use_random_agents = opt.use_random_agents or config.safe_getboolean(
"request", "random-user-agents")
opt.useragent = opt.useragent or config.safe_get("request", "user-agent")
opt.cookie = opt.cookie or config.safe_get("request", "cookie")
# Connection
opt.delay = opt.delay or config.safe_getfloat("connection", "delay")
opt.timeout = opt.timeout or config.safe_getfloat("connection", "timeout",
7.5)
opt.max_retries = opt.max_retries or config.safe_getint(
"connection", "max-retries", 1)
opt.maxrate = opt.maxrate or config.safe_getint("connection", "max-rate")
opt.proxy = opt.proxy or list(config.safe_get("connection", "proxy"))
opt.proxy_file = config.safe_get("connection", "proxy-file")
opt.scheme = opt.scheme or config.safe_get("connection", "scheme", None,
["http", "https"])
opt.replay_proxy = opt.replay_proxy or config.safe_get(
"connection", "replay-proxy")
opt.exit_on_error = opt.exit_on_error or config.safe_getboolean(
"connection", "exit-on-error")
# Output
opt.output_path = config.safe_get("output", "report-output-folder")
opt.autosave_report = config.safe_getboolean("output", "autosave-report")
opt.log_file = opt.log_file or config.safe_get("output", "log-file")
opt.output_format = opt.output_format or config.safe_get(
"output", "report-format", "plain", OUTPUT_FORMATS)
return opt
def __init__(self, script_path, arguments, output):
global VERSION
program_banner = (
open(FileUtils.build_path(script_path, "banner.txt"))
.read()
.format(**VERSION)
)
self.directories = Queue()
self.script_path = script_path
self.arguments = arguments
self.output = output
self.pass_dirs = ["/"]
if arguments.raw_file:
raw = Raw(arguments.raw_file, arguments.scheme)
self.url_list = [raw.url]
self.httpmethod = raw.method
self.data = raw.body
self.headers = raw.headers
else:
default_headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36",
"Accept-Language": "*",
"Accept-Encoding": "*",
"Keep-Alive": "timeout=15, max=1000",
"Cache-Control": "max-age=0",
}
self.url_list = arguments.url_list
self.httpmethod = arguments.httpmethod.lower()
self.data = arguments.data
self.headers = {**default_headers, **arguments.headers}
if arguments.cookie:
self.headers["Cookie"] = arguments.cookie
if arguments.useragent:
self.headers["User-Agent"] = arguments.useragent
self.recursion_depth = arguments.recursion_depth
if arguments.logs_location and self.validate_path(arguments.logs_location):
self.logs_path = FileUtils.build_path(arguments.logs_location)
elif self.validate_path(self.script_path):
self.logs_path = FileUtils.build_path(self.script_path, "logs")
if not FileUtils.exists(self.logs_path):
FileUtils.create_directory(self.logs_path)
if arguments.output_location and self.validate_path(arguments.output_location):
self.report_path = FileUtils.build_path(arguments.output_location)
elif self.validate_path(self.script_path):
self.report_path = FileUtils.build_path(self.script_path, "reports")
if not FileUtils.exists(self.report_path):
FileUtils.create_directory(self.report_path)
self.blacklists = Dictionary.generate_blacklists(arguments.extensions, self.script_path)
self.extensions = arguments.extensions
self.prefixes = arguments.prefixes
self.suffixes = arguments.suffixes
self.threads_count = arguments.threads_count
self.output_file = arguments.output_file
self.output_format = arguments.output_format
self.include_status_codes = arguments.include_status_codes
self.exclude_status_codes = arguments.exclude_status_codes
self.exclude_sizes = arguments.exclude_sizes
self.exclude_texts = arguments.exclude_texts
self.exclude_regexps = arguments.exclude_regexps
self.exclude_redirects = arguments.exclude_redirects
self.replay_proxy = arguments.replay_proxy
self.recursive = self.arguments.recursive
self.deep_recursive = arguments.deep_recursive
self.force_recursive = arguments.force_recursive
self.recursion_status_codes = arguments.recursion_status_codes
self.minimum_response_size = arguments.minimum_response_size
self.maximum_response_size = arguments.maximum_response_size
self.scan_subdirs = arguments.scan_subdirs
self.exclude_subdirs = arguments.exclude_subdirs
self.full_url = arguments.full_url
self.skip_on_status = arguments.skip_on_status
self.exit_on_error = arguments.exit_on_error
self.maxtime = arguments.maxtime
self.dictionary = Dictionary(
paths=arguments.wordlist,
extensions=arguments.extensions,
suffixes=arguments.suffixes,
prefixes=arguments.prefixes,
lowercase=arguments.lowercase,
uppercase=arguments.uppercase,
capitalization=arguments.capitalization,
force_extensions=arguments.force_extensions,
exclude_extensions=arguments.exclude_extensions,
no_extension=arguments.no_extension,
only_selected=arguments.only_selected
)
self.jobs_count = len(self.url_list) * (
len(self.scan_subdirs) if self.scan_subdirs else 1
)
self.current_job = 0
self.error_log = None
self.error_log_path = None
self.threads_lock = threading.Lock()
self.batch = False
self.batch_session = None
self.report_manager = EmptyReportManager()
self.report = EmptyReport()
self.timer = EmptyTimer()
self.output.header(program_banner)
self.print_config()
if arguments.use_random_agents:
self.random_agents = FileUtils.get_lines(
FileUtils.build_path(script_path, "db", "user-agents.txt")
)
if arguments.autosave_report or arguments.output_file:
self.setup_reports()
self.setup_error_logs()
self.output.error_log_file(self.error_log_path)
if self.maxtime:
threading.Thread(target=self.time_monitor, daemon=True).start()
try:
for url in self.url_list:
try:
gc.collect()
url = url if url.endswith("/") else url + "/"
self.output.set_target(url, arguments.scheme)
try:
self.requester = Requester(
url,
max_pool=arguments.threads_count,
max_retries=arguments.max_retries,
timeout=arguments.timeout,
ip=arguments.ip,
proxy=arguments.proxy,
proxylist=arguments.proxylist,
redirect=arguments.redirect,
request_by_hostname=arguments.request_by_hostname,
httpmethod=self.httpmethod,
data=self.data,
scheme=arguments.scheme,
)
for key, value in self.headers.items():
self.requester.set_header(key, value)
if arguments.auth:
self.requester.set_auth(arguments.auth_type, arguments.auth)
# Test request to see if server is up
self.requester.request("")
if arguments.autosave_report or arguments.output_file:
self.report = Report(self.requester.host, self.requester.port, self.requester.protocol, self.requester.base_path)
except RequestException as e:
self.output.error(e.args[0]["message"])
raise SkipTargetInterrupt
if arguments.use_random_agents:
self.requester.set_random_agents(self.random_agents)
# Initialize directories Queue with start Path
self.base_path = self.requester.base_path
self.status_skip = None
if not self.scan_subdirs:
self.directories.put("")
for subdir in self.scan_subdirs:
self.directories.put(subdir)
self.pass_dirs.append(subdir)
match_callbacks = [self.match_callback]
not_found_callbacks = [self.not_found_callback]
error_callbacks = [self.error_callback, self.append_error_log]
self.fuzzer = Fuzzer(
self.requester,
self.dictionary,
suffixes=arguments.suffixes,
prefixes=arguments.prefixes,
exclude_content=arguments.exclude_content,
threads=arguments.threads_count,
delay=arguments.delay,
maxrate=arguments.maxrate,
match_callbacks=match_callbacks,
not_found_callbacks=not_found_callbacks,
error_callbacks=error_callbacks,
)
try:
self.prepare()
except RequestException as e:
self.output.error(e.args[0]["message"])
raise SkipTargetInterrupt
except SkipTargetInterrupt:
self.report.completed = True
continue
except KeyboardInterrupt:
self.output.error("\nCanceled by the user")
exit(0)
finally:
self.error_log.close()
self.output.warning("\nTask Completed")