def write_log(self, msg): if not self.options["log_file"]: return line = time.strftime("[%y-%m-%d %H:%M:%S] ") line += msg + NEW_LINE FileUtils.write_lines(self.options["log_file"], line)
def log(file, tag, msg): if not file: return line = time.strftime("[%y-%m-%d %H:%M:%S]") line += f"[{tag.upper()}] {msg}" line += NEW_LINE FileUtils.write_lines(file, line)
def validate_dir(self, path): if not FileUtils.exists(path): self.validate_output_loc(FileUtils.parent(path)) if not FileUtils.is_dir(path): self.output.error( "{0} is a file, should be a directory".format(path)) exit(1) if not FileUtils.can_write(path): self.output.error("Directory {0} is not writable".format(path)) exit(1)
def setup(self, options): self.options = options self.pass_dirs = [''] if options["raw_file"]: self.options.update( zip(["urls", "httpmethod", "headers", "data"], parse_raw(options["raw_file"]))) else: self.options["headers"] = {**DEFAULT_HEADERS, **options["headers"]} if options["cookie"]: self.options["headers"]["Cookie"] = options["cookie"] if options["useragent"]: self.options["headers"]["User-Agent"] = options["useragent"] self.random_agents = None if options["use_random_agents"]: self.random_agents = FileUtils.get_lines( FileUtils.build_path(SCRIPT_PATH, "db", "user-agents.txt")) self.targets.queue = deque(options["urls"]) self.blacklists = Dictionary.generate_blacklists(options["extensions"]) self.dictionary = Dictionary( paths=options["wordlist"], extensions=options["extensions"], suffixes=options["suffixes"], prefixes=options["prefixes"], lowercase=options["lowercase"], uppercase=options["uppercase"], capitalization=options["capitalization"], force_extensions=options["force_extensions"], exclude_extensions=options["exclude_extensions"], no_extension=options["no_extension"], only_selected=options["only_selected"]) self.current_job = 0 self.batch = False self.batch_session = None self.exit = None self.start_time = time.time() self.jobs_count = self.targets.qsize() * (len( options["scan_subdirs"]) if options["scan_subdirs"] else 1) if options["autosave_report"] or options["output_file"]: if options["autosave_report"]: self.report_path = options[ "output_location"] or FileUtils.build_path( SCRIPT_PATH, "reports") self.create_dir(self.report_path) if options["log_file"]: options["log_file"] = FileUtils.get_abs_path(options["log_file"]) self.create_dir(FileUtils.parent(options["log_file"]))
def validate_path(self, path): if not FileUtils.exists(path): self.output.error("{0} does not exist".format(path)) exit(1) if FileUtils.exists(path) and not FileUtils.is_dir(path): self.output.error("{0} is a file, should be a directory".format(path)) exit(1) if not FileUtils.can_write(path): self.output.error("Directory {0} is not writable".format(path)) exit(1) return True
def setup_batch_reports(self): """Create batch report folder""" self.batch = True current_time = time.strftime("%y-%m-%d_%H-%M-%S") batch_session = f"BATCH-{current_time}" batch_directory_path = FileUtils.build_path(self.report_path, batch_session) try: FileUtils.create_dir(batch_directory_path) except Exception: self.output.error(f"Couldn't create batch folder at {batch_directory_path}") exit(1) return batch_directory_path
def generate_blacklists(extensions): blacklists = {} for status in [400, 403, 500]: blacklist_file_name = FileUtils.build_path(SCRIPT_PATH, "db") blacklist_file_name = FileUtils.build_path( blacklist_file_name, "{}_blacklist.txt".format(status)) if not FileUtils.can_read(blacklist_file_name): # Skip if cannot read file continue blacklists[status] = list( Dictionary([blacklist_file_name], extensions)) return blacklists
def setup_batch_reports(self): self.batch = True if not self.options["output_file"]: self.batch_session = "BATCH-{0}".format( time.strftime("%y-%m-%d_%H-%M-%S")) self.batch_directory_path = FileUtils.build_path( self.report_path, self.batch_session) if not FileUtils.exists(self.batch_directory_path): FileUtils.create_directory(self.batch_directory_path) if not FileUtils.exists(self.batch_directory_path): self.output.error( "Couldn't create batch folder at {}".format( self.batch_directory_path)) exit(1)
def __init__(self, options, output): self.targets = Queue() self.directories = Queue() self.threads_lock = threading.Lock() self.report_manager = EmptyReportManager() self.report = EmptyReport() self.output = output if options["session_file"]: self._import(FileUtils.read(options["session_file"])) self.from_export = True else: self.setup(options) self.from_export = False self.output.header(BANNER) self.output.config( ', '.join(self.options["extensions"]), ', '.join(self.options["prefixes"]), ', '.join(self.options["suffixes"]), str(self.options["threads_count"]), str(len(self.dictionary)), str(self.options["httpmethod"]), ) self.setup_reports() self.output.log_file(options["log_file"]) try: self.run() except KeyboardInterrupt: self.close("Canceled by the user")
def setup_reports(self): if self.output_file: output_file = FileUtils.get_abs_path(self.output_file) self.output.output_file(output_file) else: if len(self.url_list) > 1: self.setup_batch_reports() filename = "BATCH" filename += self.get_output_extension() directory_path = self.batch_directory_path else: parsed = urlparse(self.url_list[0]) filename = ( "{}_".format(parsed.path) ) filename += time.strftime("%y-%m-%d_%H-%M-%S") filename += self.get_output_extension() directory_path = FileUtils.build_path( self.report_path, clean_filename(parsed.netloc) ) filename = clean_filename(filename) output_file = FileUtils.build_path(directory_path, filename) if FileUtils.exists(output_file): i = 2 while FileUtils.exists(output_file + "_" + str(i)): i += 1 output_file += "_" + str(i) if not FileUtils.exists(directory_path): FileUtils.create_directory(directory_path) if not FileUtils.exists(directory_path): self.output.error( "Couldn't create the reports folder at {}".format(directory_path) ) sys.exit(1) self.output.output_file(output_file) if self.output_file and self.output_format: self.report_manager = ReportManager(self.output_format, self.output_file) elif self.output_format: self.report_manager = ReportManager(self.output_format, output_file) else: self.report_manager = ReportManager("plain", output_file)
def _export(self, session_file): self.targets.queue.insert(0, self.url) self.directories.queue.insert(0, self.current_directory) # Queue() objects, convert them to list for item in ("targets", "directories"): self.__dict__[item] = list(self.__dict__[item].queue) self.dictionary, self.dictionary_index = self.dictionary.export() self.last_output = self.output.export() self.current_job -= 1 data = { k: v for k, v in self.__dict__.items() if k not in EXCLUDED_EXPORT_VARIABLES } FileUtils.write_lines(session_file, str(data), overwrite=True)
def setup_error_logs(self): file_name = "errors-{0}.log".format(time.strftime("%y-%m-%d_%H-%M-%S")) self.error_log_path = FileUtils.build_path(self.logs_path, file_name) try: self.error_log = open(self.error_log_path, "w") except PermissionError: self.output.error( "Couldn't create the error log. Try running again with highest permission" ) sys.exit(1)
def setup_reports(self): if self.options["output_file"]: output_file = FileUtils.get_abs_path(self.options["output_file"]) self.output.output_file(output_file) else: if self.targets.qsize() > 1: self.setup_batch_reports() filename = "BATCH" filename += self.get_output_extension() directory_path = self.batch_directory_path else: parsed = urlparse(self.targets.queue[0]) filename = ("{}_".format(parsed.path)) filename += time.strftime("%y-%m-%d_%H-%M-%S") filename += self.get_output_extension() directory_path = FileUtils.build_path( self.report_path, get_valid_filename(parsed.netloc)) filename = get_valid_filename(filename) output_file = FileUtils.build_path(directory_path, filename) if FileUtils.exists(output_file): i = 2 while FileUtils.exists(output_file + "_" + str(i)): i += 1 output_file += "_" + str(i) if not FileUtils.exists(directory_path): FileUtils.create_directory(directory_path) if not FileUtils.exists(directory_path): self.output.error( "Couldn't create the reports folder at {}".format( directory_path)) exit(1) self.output.output_file(output_file) if self.options["output_format"]: self.report_manager = ReportManager( self.options["output_format"], self.options["output_file"] or output_file) else: self.report_manager = ReportManager("plain", output_file)
def generate_blacklists(extensions, script_path): reext = re.compile(r"\%ext\%", re.IGNORECASE).sub blacklists = {} for status in [400, 403, 500]: blacklist_file_name = FileUtils.build_path(script_path, "db") blacklist_file_name = FileUtils.build_path( blacklist_file_name, "{}_blacklist.txt".format(status) ) if not FileUtils.can_read(blacklist_file_name): # Skip if cannot read file continue blacklists[status] = [] for line in FileUtils.get_lines(blacklist_file_name): # Skip comments if line.lstrip().startswith("#"): continue if line.startswith("/"): line = line[1:] # Classic dirsearch blacklist processing (with %EXT% keyword) if "%ext%" in line.lower(): for extension in extensions: entry = reext.sub(extension, line) blacklists[status].append(entry) # Forced extensions is not used here because -r is only used for wordlist, # applying in blacklist may create false negatives else: blacklists[status].append(line) return blacklists
def create_dir(self, path): if path == '/': return if not FileUtils.exists(path): self.create_dir(FileUtils.parent(path)) if not FileUtils.is_dir(path): self.output.error( "{0} is a file, should be a directory".format(path)) exit(1) if not FileUtils.can_write(path): self.output.error("Directory {0} is not writable".format(path)) exit(1) FileUtils.create_directory(path)
def setup_reports(self): """Create report file""" output_file = None if self.options.output_file: output_file = FileUtils.get_abs_path(self.options.output_file) elif self.options.autosave_report: if len(self.targets) > 1: directory_path = self.setup_batch_reports() filename = "BATCH" + self.get_output_extension() else: parsed = urlparse(self.options.urls[0]) filename = get_valid_filename(f"{parsed.path}_") filename += time.strftime("%y-%m-%d_%H-%M-%S") filename += self.get_output_extension() directory_path = FileUtils.build_path( self.report_path, get_valid_filename(parsed.netloc) ) output_file = FileUtils.build_path(directory_path, filename) if FileUtils.exists(output_file): i = 2 while FileUtils.exists(f"{output_file}_{i}"): i += 1 output_file += f"_{i}" try: FileUtils.create_dir(directory_path) except Exception: self.output.error( f"Couldn't create the reports folder at {directory_path}" ) exit(1) self.report_manager = ReportManager(self.options.output_format, output_file) if output_file: self.output.output_file(output_file)
def setup(self, options, output): self.options, self.output = options, output if self.options.raw_file: self.options.update( zip( ["urls", "httpmethod", "headers", "data"], parse_raw(self.options.raw_file), ) ) else: self.options.headers = {**DEFAULT_HEADERS, **self.options.headers} if self.options.cookie: self.options.headers["Cookie"] = self.options.cookie if self.options.useragent: self.options.headers["User-Agent"] = self.options.useragent self.random_agents = None if self.options.use_random_agents: self.random_agents = FileUtils.get_lines( FileUtils.build_path(SCRIPT_PATH, "db", "user-agents.txt") ) self.requester = Requester( max_pool=self.options.threads_count, max_retries=self.options.max_retries, timeout=self.options.timeout, ip=self.options.ip, proxy=self.options.proxy, follow_redirects=self.options.follow_redirects, httpmethod=self.options.httpmethod, headers=self.options.headers, data=self.options.data, scheme=self.options.scheme, random_agents=self.random_agents, ) self.dictionary = Dictionary( paths=self.options.wordlist, extensions=self.options.extensions, suffixes=self.options.suffixes, prefixes=self.options.prefixes, lowercase=self.options.lowercase, uppercase=self.options.uppercase, capitalization=self.options.capitalization, force_extensions=self.options.force_extensions, exclude_extensions=self.options.exclude_extensions, no_extension=self.options.no_extension, only_selected=self.options.only_selected, ) self.blacklists = Dictionary.generate_blacklists(self.options.extensions) self.targets = options.urls self.start_time = time.time() self.passed_urls = set() self.directories = [] self.report = None self.batch = False self.current_job = 0 self.jobs_count = 0 self.errors = 0 self.consecutive_errors = 0 if self.options.auth: self.requester.set_auth(self.options.auth_type, self.options.auth) if self.options.proxy_auth: self.requester.set_proxy_auth(self.options.proxy_auth) if self.options.log_file: self.options.log_file = FileUtils.get_abs_path(self.options.log_file) try: FileUtils.create_dir(FileUtils.parent(self.options.log_file)) if not FileUtils.can_write(self.options.log_file): raise Exception except Exception: self.output.error( f"Couldn't create log file at {self.options.log_file}" ) exit(1) if self.options.autosave_report: self.report_path = self.options.output_path or FileUtils.build_path( SCRIPT_PATH, "reports" ) try: FileUtils.create_dir(self.report_path) if not FileUtils.can_write(self.report_path): raise Exception except Exception: self.output.error( f"Couldn't create report folder at {self.report_path}" ) exit(1) self.output.header(BANNER) self.output.config( ", ".join(self.options["extensions"]), ", ".join(self.options["prefixes"]), ", ".join(self.options["suffixes"]), str(self.options["threads_count"]), str(len(self.dictionary)), str(self.options["httpmethod"]), ) self.setup_reports() if self.options.log_file: self.output.log_file(self.options.log_file)
def parse_arguments(): usage = "Usage: %prog [-u|--url] target [-e|--extensions] extensions [options]" parser = OptionParser(usage, version="dirsearch v{}".format(VERSION)) # Mandatory arguments mandatory = OptionGroup(parser, "Mandatory") mandatory.add_option("-u", "--url", action="append", dest="urls", metavar="URL", help="Target URL(s), support multiple flags") mandatory.add_option("-l", "--url-list", action="store", dest="url_list", metavar="FILE", help="URL list file") mandatory.add_option("--stdin", action="store_true", dest="stdin_urls", help="Read URL(s) from STDIN") mandatory.add_option("--cidr", action="store", dest="cidr", help="Target CIDR") mandatory.add_option( "--raw", action="store", dest="raw_file", metavar="FILE", help= "Load raw HTTP request from file (use `--scheme` flag to set the scheme)" ) mandatory.add_option("-s", "--session", action="store", dest="session_file", help="Session file") mandatory.add_option( "-e", "--extensions", action="store", dest="extensions", help="Extension list separated by commas (Example: php,asp)") mandatory.add_option( "-X", "--exclude-extensions", action="store", dest="exclude_extensions", metavar="EXTENSIONS", help="Exclude extension list separated by commas (Example: asp,jsp)") mandatory.add_option( "-f", "--force-extensions", action="store_true", dest="force_extensions", help= "Add extensions to every wordlist entry. By default dirsearch only replaces the %EXT% keyword with extensions" ) mandatory.add_option( "--config", action="store", dest="config", default=FileUtils.build_path(SCRIPT_PATH, "default.conf"), metavar="FILE", help= "Full path to config file, see 'default.conf' for example (Default: default.conf)" ) # Dictionary Settings dictionary = OptionGroup(parser, "Dictionary Settings") dictionary.add_option("-w", "--wordlists", action="store", dest="wordlist", help="Customize wordlists (separated by commas)") dictionary.add_option( "--prefixes", action="store", dest="prefixes", help="Add custom prefixes to all wordlist entries (separated by commas)" ) dictionary.add_option( "--suffixes", action="store", dest="suffixes", help= "Add custom suffixes to all wordlist entries, ignore directories (separated by commas)" ) dictionary.add_option( "--only-selected", action="store_true", dest="only_selected", help= "Remove paths have different extensions from selected ones via `-e` (keep entries don't have extensions)" ) dictionary.add_option( "--remove-extensions", action="store_true", dest="no_extension", help="Remove extensions in all paths (Example: admin.php -> admin)") dictionary.add_option("-U", "--uppercase", action="store_true", dest="uppercase", help="Uppercase wordlist") dictionary.add_option("-L", "--lowercase", action="store_true", dest="lowercase", help="Lowercase wordlist") dictionary.add_option("-C", "--capital", action="store_true", dest="capitalization", help="Capital wordlist") # Optional Settings general = OptionGroup(parser, "General Settings") general.add_option("-t", "--threads", action="store", type="int", dest="threads_count", metavar="THREADS", help="Number of threads") general.add_option("-r", "--recursive", action="store_true", dest="recursive", help="Brute-force recursively") general.add_option( "--deep-recursive", action="store_true", dest="deep_recursive", help= "Perform recursive scan on every directory depth (Example: api/users -> api/)" ) general.add_option( "--force-recursive", action="store_true", dest="force_recursive", help= "Do recursive brute-force for every found path, not only directories") general.add_option("-R", "--max-recursion-depth", action="store", type="int", dest="recursion_depth", metavar="DEPTH", help="Maximum recursion depth") general.add_option( "--recursion-status", action="store", dest="recursion_status_codes", metavar="CODES", help= "Valid status codes to perform recursive scan, support ranges (separated by commas)" ) general.add_option( "--subdirs", action="store", dest="scan_subdirs", metavar="SUBDIRS", help="Scan sub-directories of the given URL[s] (separated by commas)") general.add_option( "--exclude-subdirs", action="store", dest="exclude_subdirs", metavar="SUBDIRS", help= "Exclude the following subdirectories during recursive scan (separated by commas)" ) general.add_option( "-i", "--include-status", action="store", dest="include_status_codes", metavar="CODES", help= "Include status codes, separated by commas, support ranges (Example: 200,300-399)" ) general.add_option( "-x", "--exclude-status", action="store", dest="exclude_status_codes", metavar="CODES", help= "Exclude status codes, separated by commas, support ranges (Example: 301,500-599)" ) general.add_option( "--exclude-sizes", action="store", dest="exclude_sizes", metavar="SIZES", help="Exclude responses by sizes, separated by commas (Example: 0B,4KB)" ) general.add_option( "--exclude-texts", action="store", dest="exclude_texts", metavar="TEXTS", help= "Exclude responses by texts, separated by commas (Example: 'Not found', 'Error')" ) general.add_option("--exclude-regex", action="store", dest="exclude_regex", metavar="REGEX", help="Exclude responses by regex (Example: '^Error$')") general.add_option( "--exclude-redirect", action="store", dest="exclude_redirect", metavar="STRING", help= "Exclude responses if this regex (or text) matches redirect URL (Example: '/index.html')", ) general.add_option( "--exclude-response", action="store", dest="exclude_response", metavar="PATH", help= "Exclude responses similar to response of this page, path as input (Example: 404.html)" ) general.add_option( "--skip-on-status", action="store", dest="skip_on_status", metavar="CODES", help= "Skip target whenever hit one of these status codes, separated by commas, support ranges" ) general.add_option("--min-response-size", action="store", type="int", dest="minimum_response_size", help="Minimum response length", metavar="LENGTH", default=0) general.add_option("--max-response-size", action="store", type="int", dest="maximum_response_size", help="Maximum response length", metavar="LENGTH", default=0) general.add_option("--redirects-history", action="store_true", dest="redirects_history", help="Show redirects history") general.add_option("--max-time", action="store", type="int", dest="maxtime", metavar="SECONDS", help="Maximum runtime for the scan") general.add_option( "--full-url", action="store_true", dest="full_url", help="Full URLs in the output (enabled automatically in quiet mode)") general.add_option("--no-color", action="store_false", dest="color", help="No colored output") general.add_option("-q", "--quiet-mode", action="store_true", dest="quiet", help="Quiet mode") # Request Settings request = OptionGroup(parser, "Request Settings") request.add_option("-m", "--http-method", action="store", dest="httpmethod", metavar="METHOD", help="HTTP method (default: GET)") request.add_option("-d", "--data", action="store", dest="data", help="HTTP request data") request.add_option("-H", "--header", action="append", dest="headers", help="HTTP request header, support multiple flags") request.add_option("--header-list", dest="header_list", metavar="FILE", help="File contains HTTP request headers") request.add_option("-F", "--follow-redirects", action="store_true", dest="follow_redirects", help="Follow HTTP redirects") request.add_option("--random-agent", action="store_true", dest="use_random_agents", help="Choose a random User-Agent for each request") request.add_option("--auth-type", action="store", dest="auth_type", metavar="TYPE", help="Authentication type ({})".format( ", ".join(AUTHENTICATION_TYPES))) request.add_option( "--auth", action="store", dest="auth", metavar="CREDENTIAL", help="Authentication credential ([user]:[password] or bearer token)") request.add_option("--user-agent", action="store", dest="useragent") request.add_option("--cookie", action="store", dest="cookie") # Connection Settings connection = OptionGroup(parser, "Connection Settings") connection.add_option("--timeout", action="store", type="float", dest="timeout", help="Connection timeout") connection.add_option("--delay", action="store", type="float", dest="delay", help="Delay between requests") connection.add_option( "--proxy", action="store", dest="proxy", metavar="PROXY", help= "Proxy URL, support HTTP and SOCKS proxies (Example: localhost:8080, socks5://localhost:8088)" ) connection.add_option("--proxy-list", action="store", type="string", dest="proxylist", help="File contains proxy servers", metavar="FILE") connection.add_option("--replay-proxy", action="store", dest="replay_proxy", metavar="PROXY", help="Proxy to replay with found paths") connection.add_option( "--scheme", action="store", dest="scheme", metavar="SCHEME", help= "Scheme for raw request or if there is no scheme in the URL (Default: auto-detect)" ) connection.add_option("--max-rate", action="store", type="int", dest="maxrate", metavar="RATE", help="Max requests per second") connection.add_option("--retries", action="store", type="int", dest="max_retries", metavar="RETRIES", help="Number of retries for failed requests") connection.add_option( "-b", "--request-by-hostname", action="store_true", dest="request_by_hostname", help= "By default dirsearch requests by IP for speed. This will force dirsearch to request by hostname" ) connection.add_option("--ip", action="store", dest="ip", help="Server IP address") connection.add_option("--exit-on-error", action="store_true", dest="exit_on_error", help="Exit whenever an error occurs") # Output Settings output = OptionGroup(parser, "Output") output.add_option("-o", "--output", action="store", dest="output_file", metavar="FILE", help="Output file") output.add_option( "--format", action="store", dest="output_format", metavar="FORMAT", help= "Report format (Available: simple, plain, json, xml, md, csv, html, sqlite)" ) output.add_option("--log", action="store", dest="log_file", metavar="FILE", help="Log file") parser.add_option_group(mandatory) parser.add_option_group(dictionary) parser.add_option_group(general) parser.add_option_group(request) parser.add_option_group(connection) parser.add_option_group(output) options, arguments = parser.parse_args() return options
def parse_config(self): config = ConfigParser() config_path = FileUtils.build_path(self.script_path, "default.conf") config.read(config_path) # Mandatory self.default_extensions = config.safe_get("mandatory", "default-extensions", str()) self.exclude_extensions = config.safe_get("mandatory", "exclude-extensions", None) self.force_extensions = config.safe_getboolean("mandatory", "force-extensions", False) # General self.threads_count = config.safe_getint("general", "threads", 30, list(range(1, 300))) self.include_status_codes = config.safe_get("general", "include-status", None) self.exclude_status_codes = config.safe_get("general", "exclude-status", None) self.exclude_sizes = config.safe_get("general", "exclude-sizes", None) self.exclude_texts = config.safe_get("general", "exclude-texts", None) self.exclude_regexps = config.safe_get("general", "exclude-regexps", None) self.exclude_redirects = config.safe_get("general", "exclude-redirects", None) self.exclude_content = config.safe_get("general", "exclude-content", "") self.recursive = config.safe_getboolean("general", "recursive", False) self.deep_recursive = config.safe_getboolean("general", "deep-recursive", False) self.force_recursive = config.safe_getboolean("general", "force-recursive", False) self.recursion_depth = config.safe_getint("general", "recursion-depth", 0) self.recursion_status_codes = config.safe_get("general", "recursion-status", None) self.exclude_subdirs = config.safe_get("general", "exclude-subdirs", None) self.skip_on_status = config.safe_get("general", "skip-on-status", None) self.maxtime = config.safe_getint("general", "max-time", 0) self.full_url = config.safe_getboolean("general", "full-url", False) self.color = config.safe_getboolean("general", "color", True) self.quiet = config.safe_getboolean("general", "quiet-mode", False) # Reports self.output_location = config.safe_get("reports", "report-output-folder", None) self.autosave_report = config.safe_getboolean("reports", "autosave-report", False) self.logs_location = config.safe_get("reports", "logs-location", None) self.output_format = config.safe_get( "reports", "report-format", "plain", ["plain", "simple", "json", "xml", "md", "csv", "html"]) # Dictionary self.wordlist = config.safe_get( "dictionary", "wordlist", FileUtils.build_path(self.script_path, "db", "dicc.txt"), ) self.prefixes = config.safe_get("dictionary", "prefixes", None) self.suffixes = config.safe_get("dictionary", "suffixes", None) self.lowercase = config.safe_getboolean("dictionary", "lowercase", False) self.uppercase = config.safe_getboolean("dictionary", "uppercase", False) self.capitalization = config.safe_getboolean("dictionary", "capitalization", False) # Request self.httpmethod = config.safe_get("request", "httpmethod", "get") self.header_list = config.safe_get("request", "headers-file", None) self.redirect = config.safe_getboolean("request", "follow-redirects", False) self.use_random_agents = config.safe_get("request", "random-user-agents", False) self.useragent = config.safe_get("request", "user-agent", "") self.cookie = config.safe_get("request", "cookie", "") # Connection self.delay = config.safe_getfloat("connection", "delay", 0) self.timeout = config.safe_getint("connection", "timeout", 10) self.max_retries = config.safe_getint("connection", "retries", 2) self.maxrate = config.safe_getint("connection", "max-rate", 0) self.proxy = config.safe_get("connection", "proxy", None) self.proxylist = config.safe_get("connection", "proxy-list", None) self.scheme = config.safe_get("connection", "scheme", "http", ["http", "https"]) self.replay_proxy = config.safe_get("connection", "replay-proxy", None) self.request_by_hostname = config.safe_getboolean( "connection", "request-by-hostname", False) self.exit_on_error = config.safe_getboolean("connection", "exit-on-error", False)
import os import sys import string from lib.utils.file import FileUtils # Version format: <major version>.<minor version>.<revision>[.<month>] VERSION = "0.4.2.4" BANNER = f""" _|. _ _ _ _ _ _|_ v{VERSION} (_||| _) (/_(_|| (_| ) """ SCRIPT_PATH = FileUtils.parent(__file__, 3) IS_WINDOWS = sys.platform in ("win32", "msys") DEFAULT_ENCODING = "utf-8" NEW_LINE = os.linesep INVALID_CHARS_FOR_WINDOWS_FILENAME = ('"', "*", "<", ">", "?", "\\", "|", "/", ":") INVALID_FILENAME_CHAR_REPLACEMENT = "_" OUTPUT_FORMATS = ("simple", "plain", "json", "xml", "md", "csv", "html", "sqlite") COMMON_EXTENSIONS = ("php", "jsp", "asp", "aspx", "do", "action", "cgi", "html", "htm", "js", "json", "tar.gz", "bak")
def get_dependencies(): try: return FileUtils.get_lines(REQUIREMENTS_FILE) except FileNotFoundError: print("Can't find requirements.txt") exit(1)
def generate(self): """ Dictionary.generate() behaviour Classic dirsearch wordlist: 1. If %EXT% keyword is present, append one with each extension REPLACED. 2. If the special word is no present, append line unmodified. Forced extensions wordlist (NEW): This type of wordlist processing is a mix between classic processing and DirBuster processing. 1. If %EXT% keyword is present in the line, immediately process as "classic dirsearch" (1). 2. If the line does not include the special word AND is NOT terminated by a slash, append one with each extension APPENDED (line.ext) and ONLYE ONE with a slash. 3. If the line does not include the special word and IS ALREADY terminated by slash, append line unmodified. """ reext = re.compile(EXTENSION_TAG, re.IGNORECASE) result = [] # Enable to use multiple dictionaries at once for dict_file in self._dictionary_files: for line in FileUtils.get_lines(dict_file): if line.startswith("/"): line = line[1:] if self.no_extension: line = line.split(".")[0] # Skip comments if line.lstrip().startswith("#"): continue # Skip if the path contains excluded extensions if any("." + extension in line for extension in self.exclude_extensions): continue # Classic dirsearch wordlist processing (with %EXT% keyword) if EXTENSION_TAG in line.lower(): for extension in self.extensions: newline = reext.sub(extension, line) result.append(newline) # If "forced extensions" is used and the path is not a directory (terminated by /) or has # had an extension already, append extensions to the path elif (self.force_extensions and not line.endswith("/") and not re.search(EXTENSION_REGEX, line)): for extension in self.extensions: result.append(line + f".{extension}") result.append(line) result.append(line + "/") # Append line unmodified. elif not self.only_selected or any( line.endswith(f".{extension}") for extension in self.extensions): result.append(line) # Re-add dictionary with prefixes result.extend(pref + path for path in result for pref in self.prefixes if not path.startswith(pref)) # Re-add dictionary with suffixes result.extend(path + suff for path in result for suff in self.suffixes if not path.endswith(("/", suff))) if self.lowercase: self._entries = tuple(entry.lower() for entry in uniq(result)) elif self.uppercase: self._entries = tuple(entry.upper() for entry in uniq(result)) elif self.capitalization: self._entries = tuple(entry.capitalize() for entry in uniq(result)) else: self._entries = tuple(uniq(result)) del result
def __init__(self, options, output): self.directories = Queue() self.output = output self.options = options self.pass_dirs = ["/"] if options.raw_file: raw = Raw(options.raw_file) self.url_list = [raw.url] self.httpmethod = raw.method self.data = raw.body self.headers = raw.headers else: self.url_list = options.url_list self.httpmethod = options.httpmethod self.data = options.httpmethod self.headers = {**DEFAULT_HEADERS, **options.headers} if options.cookie: self.headers["Cookie"] = options.cookie if options.useragent: self.headers["User-Agent"] = options.useragent self.random_agents = None if options.use_random_agents: self.random_agents = FileUtils.get_lines( FileUtils.build_path(SCRIPT_PATH, "db", "user-agents.txt")) self.blacklists = Dictionary.generate_blacklists(options.extensions) self.dictionary = Dictionary( paths=options.wordlist, extensions=options.extensions, suffixes=options.suffixes, prefixes=options.prefixes, lowercase=options.lowercase, uppercase=options.uppercase, capitalization=options.capitalization, force_extensions=options.force_extensions, exclude_extensions=options.exclude_extensions, no_extension=options.no_extension, only_selected=options.only_selected) self.jobs_count = len(self.url_list) * (len(options.scan_subdirs) if options.scan_subdirs else 1) self.current_job = 0 self.batch = False self.batch_session = None self.exit = None self.threads_lock = threading.Lock() self.report_manager = EmptyReportManager() self.report = EmptyReport() self.start_time = time.time() self.output.header(BANNER) self.print_config() if options.autosave_report or options.output_file: if options.autosave_report: self.report_path = options.output_location or FileUtils.build_path( SCRIPT_PATH, "reports") self.validate_dir(self.report_path) self.setup_reports() if options.log_file: self.validate_dir(FileUtils.parent(options.log_file)) FileUtils.create_directory(FileUtils.parent(options.log_file)) self.output.log_file(FileUtils.get_abs_path(options.log_file)) try: for url in self.url_list: try: gc.collect() try: self.requester = Requester( url if url.endswith("/") else url + "/", max_pool=options.threads_count, max_retries=options.max_retries, timeout=options.timeout, ip=options.ip, proxy=options.proxy, proxylist=options.proxylist, redirect=options.follow_redirects, request_by_hostname=options.request_by_hostname, httpmethod=self.httpmethod, data=self.data, scheme=options.scheme, random_agents=self.random_agents, ) self.output.set_target(self.requester.base_url + self.requester.base_path) self.requester.setup() for key, value in self.headers.items(): self.requester.set_header(key, value) if options.auth: self.requester.set_auth(options.auth_type, options.auth) # Test request to check if server is up self.requester.request("") self.write_log("Test request sent for: {}".format( self.requester.base_url)) if options.autosave_report or options.output_file: self.report = Report(self.requester.host, self.requester.port, self.requester.scheme, self.requester.base_path) except RequestException as e: self.output.error(e.args[0]) raise SkipTargetInterrupt self.skip = None if not options.scan_subdirs: self.directories.put("") for subdir in options.scan_subdirs: self.directories.put(subdir) self.pass_dirs.append(subdir) match_callbacks = [self.match_callback, self.append_log] not_found_callbacks = [ self.not_found_callback, self.append_log ] error_callbacks = [ self.error_callback, self.append_error_log ] self.fuzzer = Fuzzer( self.requester, self.dictionary, suffixes=options.suffixes, prefixes=options.prefixes, exclude_response=options.exclude_response, threads=options.threads_count, delay=options.delay, maxrate=options.maxrate, match_callbacks=match_callbacks, not_found_callbacks=not_found_callbacks, error_callbacks=error_callbacks, ) try: self.prepare() except RequestException as e: self.output.error(e.args[0]) raise SkipTargetInterrupt except SkipTargetInterrupt: self.report.completed = True continue except KeyboardInterrupt: self.close("Canceled by the user") self.output.warning("\nTask Completed")
def parse_config(opt): config = ConfigParser() config.read(opt.config) # Mandatory opt.extensions = opt.extensions or config.safe_get("mandatory", "default-extensions") opt.exclude_extensions = opt.exclude_extensions or config.safe_get( "mandatory", "exclude-extensions") opt.force_extensions = opt.force_extensions or config.safe_getboolean( "mandatory", "force-extensions") # General opt.threads_count = opt.threads_count or config.safe_getint( "general", "threads", 25) opt.include_status_codes = opt.include_status_codes or config.safe_get( "general", "include-status") opt.exclude_status_codes = opt.exclude_status_codes or config.safe_get( "general", "exclude-status") opt.exclude_sizes = opt.exclude_sizes or config.safe_get( "general", "exclude-sizes") opt.exclude_texts = opt.exclude_texts or config.safe_get( "general", "exclude-texts") opt.exclude_regex = opt.exclude_regex or config.safe_get( "general", "exclude-regex") opt.exclude_redirect = opt.exclude_redirect or config.safe_get( "general", "exclude-redirect") opt.exclude_response = opt.exclude_response or config.safe_get( "general", "exclude-response") opt.recursive = opt.recursive or config.safe_getboolean( "general", "recursive") opt.deep_recursive = opt.deep_recursive or config.safe_getboolean( "general", "deep-recursive") opt.force_recursive = opt.force_recursive or config.safe_getboolean( "general", "force-recursive") opt.recursion_depth = opt.recursion_depth or config.safe_getint( "general", "max-recursion-depth") opt.recursion_status_codes = opt.recursion_status_codes or config.safe_get( "general", "recursion-status", "100-999") opt.scan_subdirs = opt.scan_subdirs or config.safe_get( "general", "subdirs") opt.exclude_subdirs = opt.exclude_subdirs or config.safe_get( "general", "exclude-subdirs") opt.skip_on_status = opt.skip_on_status or config.safe_get( "general", "skip-on-status") opt.maxtime = opt.maxtime or config.safe_getint("general", "max-time") opt.full_url = opt.full_url or config.safe_getboolean( "general", "full-url") opt.color = opt.color or config.safe_getboolean("general", "color", True) opt.quiet = opt.quiet or config.safe_getboolean("general", "quiet-mode") opt.redirects_history = opt.redirects_history or config.safe_getboolean( "general", "show-redirects-history") # Dictionary opt.wordlist = opt.wordlist or config.safe_get( "dictionary", "wordlist", FileUtils.build_path(SCRIPT_PATH, "db", "dicc.txt"), ) opt.prefixes = opt.prefixes or config.safe_get( "dictionary", "prefixes", ) opt.suffixes = opt.suffixes or config.safe_get("dictionary", "suffixes") opt.lowercase = opt.lowercase or config.safe_getboolean( "dictionary", "lowercase") opt.uppercase = opt.uppercase or config.safe_getboolean( "dictionary", "uppercase") opt.capitalization = opt.capitalization or config.safe_getboolean( "dictionary", "capitalization") # Request opt.httpmethod = opt.httpmethod or config.safe_get("request", "httpmethod", "get") opt.header_file = opt.header_file or config.safe_get( "request", "headers-file") opt.follow_redirects = opt.follow_redirects or config.safe_getboolean( "request", "follow-redirects") opt.use_random_agents = opt.use_random_agents or config.safe_getboolean( "request", "random-user-agents") opt.useragent = opt.useragent or config.safe_get("request", "user-agent") opt.cookie = opt.cookie or config.safe_get("request", "cookie") # Connection opt.delay = opt.delay or config.safe_getfloat("connection", "delay") opt.timeout = opt.timeout or config.safe_getfloat("connection", "timeout", 7.5) opt.max_retries = opt.max_retries or config.safe_getint( "connection", "max-retries", 1) opt.maxrate = opt.maxrate or config.safe_getint("connection", "max-rate") opt.proxy = opt.proxy or list(config.safe_get("connection", "proxy")) opt.proxy_file = config.safe_get("connection", "proxy-file") opt.scheme = opt.scheme or config.safe_get("connection", "scheme", None, ["http", "https"]) opt.replay_proxy = opt.replay_proxy or config.safe_get( "connection", "replay-proxy") opt.exit_on_error = opt.exit_on_error or config.safe_getboolean( "connection", "exit-on-error") # Output opt.output_path = config.safe_get("output", "report-output-folder") opt.autosave_report = config.safe_getboolean("output", "autosave-report") opt.log_file = opt.log_file or config.safe_get("output", "log-file") opt.output_format = opt.output_format or config.safe_get( "output", "report-format", "plain", OUTPUT_FORMATS) return opt
def __init__(self, script_path, arguments, output): global VERSION program_banner = ( open(FileUtils.build_path(script_path, "banner.txt")) .read() .format(**VERSION) ) self.directories = Queue() self.script_path = script_path self.arguments = arguments self.output = output self.pass_dirs = ["/"] if arguments.raw_file: raw = Raw(arguments.raw_file, arguments.scheme) self.url_list = [raw.url] self.httpmethod = raw.method self.data = raw.body self.headers = raw.headers else: default_headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36", "Accept-Language": "*", "Accept-Encoding": "*", "Keep-Alive": "timeout=15, max=1000", "Cache-Control": "max-age=0", } self.url_list = arguments.url_list self.httpmethod = arguments.httpmethod.lower() self.data = arguments.data self.headers = {**default_headers, **arguments.headers} if arguments.cookie: self.headers["Cookie"] = arguments.cookie if arguments.useragent: self.headers["User-Agent"] = arguments.useragent self.recursion_depth = arguments.recursion_depth if arguments.logs_location and self.validate_path(arguments.logs_location): self.logs_path = FileUtils.build_path(arguments.logs_location) elif self.validate_path(self.script_path): self.logs_path = FileUtils.build_path(self.script_path, "logs") if not FileUtils.exists(self.logs_path): FileUtils.create_directory(self.logs_path) if arguments.output_location and self.validate_path(arguments.output_location): self.report_path = FileUtils.build_path(arguments.output_location) elif self.validate_path(self.script_path): self.report_path = FileUtils.build_path(self.script_path, "reports") if not FileUtils.exists(self.report_path): FileUtils.create_directory(self.report_path) self.blacklists = Dictionary.generate_blacklists(arguments.extensions, self.script_path) self.extensions = arguments.extensions self.prefixes = arguments.prefixes self.suffixes = arguments.suffixes self.threads_count = arguments.threads_count self.output_file = arguments.output_file self.output_format = arguments.output_format self.include_status_codes = arguments.include_status_codes self.exclude_status_codes = arguments.exclude_status_codes self.exclude_sizes = arguments.exclude_sizes self.exclude_texts = arguments.exclude_texts self.exclude_regexps = arguments.exclude_regexps self.exclude_redirects = arguments.exclude_redirects self.replay_proxy = arguments.replay_proxy self.recursive = self.arguments.recursive self.deep_recursive = arguments.deep_recursive self.force_recursive = arguments.force_recursive self.recursion_status_codes = arguments.recursion_status_codes self.minimum_response_size = arguments.minimum_response_size self.maximum_response_size = arguments.maximum_response_size self.scan_subdirs = arguments.scan_subdirs self.exclude_subdirs = arguments.exclude_subdirs self.full_url = arguments.full_url self.skip_on_status = arguments.skip_on_status self.exit_on_error = arguments.exit_on_error self.maxtime = arguments.maxtime self.dictionary = Dictionary( paths=arguments.wordlist, extensions=arguments.extensions, suffixes=arguments.suffixes, prefixes=arguments.prefixes, lowercase=arguments.lowercase, uppercase=arguments.uppercase, capitalization=arguments.capitalization, force_extensions=arguments.force_extensions, exclude_extensions=arguments.exclude_extensions, no_extension=arguments.no_extension, only_selected=arguments.only_selected ) self.jobs_count = len(self.url_list) * ( len(self.scan_subdirs) if self.scan_subdirs else 1 ) self.current_job = 0 self.error_log = None self.error_log_path = None self.threads_lock = threading.Lock() self.batch = False self.batch_session = None self.report_manager = EmptyReportManager() self.report = EmptyReport() self.timer = EmptyTimer() self.output.header(program_banner) self.print_config() if arguments.use_random_agents: self.random_agents = FileUtils.get_lines( FileUtils.build_path(script_path, "db", "user-agents.txt") ) if arguments.autosave_report or arguments.output_file: self.setup_reports() self.setup_error_logs() self.output.error_log_file(self.error_log_path) if self.maxtime: threading.Thread(target=self.time_monitor, daemon=True).start() try: for url in self.url_list: try: gc.collect() url = url if url.endswith("/") else url + "/" self.output.set_target(url, arguments.scheme) try: self.requester = Requester( url, max_pool=arguments.threads_count, max_retries=arguments.max_retries, timeout=arguments.timeout, ip=arguments.ip, proxy=arguments.proxy, proxylist=arguments.proxylist, redirect=arguments.redirect, request_by_hostname=arguments.request_by_hostname, httpmethod=self.httpmethod, data=self.data, scheme=arguments.scheme, ) for key, value in self.headers.items(): self.requester.set_header(key, value) if arguments.auth: self.requester.set_auth(arguments.auth_type, arguments.auth) # Test request to see if server is up self.requester.request("") if arguments.autosave_report or arguments.output_file: self.report = Report(self.requester.host, self.requester.port, self.requester.protocol, self.requester.base_path) except RequestException as e: self.output.error(e.args[0]["message"]) raise SkipTargetInterrupt if arguments.use_random_agents: self.requester.set_random_agents(self.random_agents) # Initialize directories Queue with start Path self.base_path = self.requester.base_path self.status_skip = None if not self.scan_subdirs: self.directories.put("") for subdir in self.scan_subdirs: self.directories.put(subdir) self.pass_dirs.append(subdir) match_callbacks = [self.match_callback] not_found_callbacks = [self.not_found_callback] error_callbacks = [self.error_callback, self.append_error_log] self.fuzzer = Fuzzer( self.requester, self.dictionary, suffixes=arguments.suffixes, prefixes=arguments.prefixes, exclude_content=arguments.exclude_content, threads=arguments.threads_count, delay=arguments.delay, maxrate=arguments.maxrate, match_callbacks=match_callbacks, not_found_callbacks=not_found_callbacks, error_callbacks=error_callbacks, ) try: self.prepare() except RequestException as e: self.output.error(e.args[0]["message"]) raise SkipTargetInterrupt except SkipTargetInterrupt: self.report.completed = True continue except KeyboardInterrupt: self.output.error("\nCanceled by the user") exit(0) finally: self.error_log.close() self.output.warning("\nTask Completed")