def __init__( self, paths, extensions, suffixes=None, prefixes=None, lowercase=False, uppercase=False, capitalization=False, forcedExtensions=False, noDotExtensions=False, excludeExtensions=[], noExtension=False, ): self.entries = [] self.currentIndex = 0 self.condition = threading.Lock() self._extensions = extensions self._excludeExtensions = excludeExtensions self._prefixes = prefixes self._suffixes = suffixes self._paths = paths self._forcedExtensions = forcedExtensions self._noDotExtensions = noDotExtensions self._noExtension = noExtension self.lowercase = lowercase self.uppercase = uppercase self.capitalization = capitalization self.dictionaryFiles = [File(path) for path in self.paths] self.generate()
def __init__(self, script_path): self.script_path = script_path self.parseConfig() options = self.parseArguments() self.quiet = options.quiet self.full_url = options.full_url self.urlList = None self.raw_file = None if not options.url: if options.urlList: with File(options.urlList) as urlList: if not urlList.exists(): print("The file with URLs does not exist") exit(1) if not urlList.is_valid(): print("The file with URLs is invalid") exit(1) if not urlList.can_read(): print("The file with URLs cannot be read") exit(1) self.urlList = list(urlList.get_lines()) elif options.cidr: self.urlList = [str(ip) for ip in IPv4Network(options.cidr)] elif options.stdin_urls: self.urlList = sys.stdin.read().splitlines() elif options.raw_file: with File(options.raw_file) as raw_content: if not raw_content.exists(): print("The file with the raw request does not exist") exit(1) if not raw_content.is_valid(): print("The file with the raw request is invalid") exit(1) if not raw_content.can_read(): print("The file with the raw request cannot be read") exit(1) self.raw_file = options.raw_file else: print("URL target is missing, try using -u <url>") exit(1) else: self.urlList = [options.url] if not options.extensions and not options.noExtension: print("WARNING: No extension was specified!") if options.noExtension: options.extensions = str() # Enable to use multiple dictionaries at once for dictFile in options.wordlist.split(","): with File(dictFile) as wordlist: if not wordlist.exists(): print("The wordlist file does not exist") exit(1) if not wordlist.is_valid(): print("The wordlist is invalid") exit(1) if not wordlist.can_read(): print("The wordlist cannot be read") exit(1) if options.proxyList: with File(options.proxyList) as plist: if not plist.exists(): print("The proxylist file does not exist") exit(1) if not plist.is_valid(): print("The proxylist is invalid") exit(1) if not plist.can_read(): print("The proxylist cannot be read") exit(1) self.proxylist = open(options.proxyList).read().splitlines() options.requestByHostname = True elif options.proxy: self.proxy = options.proxy options.requestByHostname = True else: self.proxy = None if options.replay_proxy: self.replay_proxy = options.replay_proxy options.requestByHostname = True else: self.replay_proxy = None if options.headers: try: self.headers = dict( email.message_from_file( StringIO("\r\n".join(options.headers)))) except Exception: print("Invalid headers") exit(1) else: self.headers = {} if options.headerList: try: with File(options.headerList) as hlist: if not hlist.exists(): print("The header list file does not exist") exit(1) if not hlist.is_valid(): print("The header list file is invalid") exit(1) if not hlist.can_read(): print("The header list cannot be read") exit(1) headers = dict( email.message_from_file(StringIO(hlist.read()))) for key, value in headers.items(): self.headers[key] = value except Exception as e: print("Error in headers file: " + str(e)) exit(1) if options.extensions == "*": self.extensions = [ "php", "jsp", "jsf", "asp", "aspx", "do", "action", "cgi", "pl", "html", "htm", "js", "json", "json", "tar.gz", "tgz" ] elif options.extensions == "CHANGELOG.md": print( "A weird extension was provided: CHANGELOG.md. Please do not use * as the extension or enclose it in double quotes" ) exit(0) else: self.extensions = list( oset([ extension.lstrip(' .') for extension in options.extensions.split(",") ])) if options.excludeExtensions: self.excludeExtensions = list( oset([ excludeExtension.lstrip(' .') for excludeExtension in options.excludeExtensions.split(",") ])) else: self.excludeExtensions = [] self.useragent = options.useragent self.useRandomAgents = options.useRandomAgents self.cookie = options.cookie if options.threadsCount < 1: print("Threads number must be greater than zero") exit(1) self.threadsCount = options.threadsCount self.includeStatusCodes = [] if options.includeStatusCodes: for statusCode in options.includeStatusCodes.split(","): try: if "-" in statusCode: statusCodes = [ i for i in range( int(statusCode.split("-")[0].strip()), int(statusCode.split("-")[1].strip()) + 1) ] self.includeStatusCodes.extend(statusCodes) else: self.includeStatusCodes.append(int(statusCode.strip())) except ValueError: print( "Invalid status code or status code range: {0}".format( statusCode)) exit(1) self.excludeStatusCodes = [] if options.excludeStatusCodes: for statusCode in options.excludeStatusCodes.split(","): try: if "-" in statusCode: statusCodes = [ i for i in range( int(statusCode.split("-")[0].strip()), int(statusCode.split("-")[1].strip()) + 1) ] self.excludeStatusCodes.extend(statusCodes) else: self.excludeStatusCodes.append(int(statusCode.strip())) except ValueError: print( "Invalid status code or status code range: {0}".format( statusCode)) exit(1) self.recursionStatusCodes = [] if options.recursionStatusCodes: for statusCode in options.recursionStatusCodes.split(","): try: if "-" in statusCode: statusCodes = [ i for i in range( int(statusCode.split("-")[0].strip()), int(statusCode.split("-")[1].strip()) + 1) ] self.recursionStatusCodes.extend(statusCodes) else: self.recursionStatusCodes.append( int(statusCode.strip())) except ValueError: print( "Invalid status code or status code range: {0}".format( statusCode)) exit(1) if options.excludeSizes: try: self.excludeSizes = list( oset([ excludeSize.strip().upper() if excludeSize else None for excludeSize in options.excludeSizes.split(",") ])) except ValueError: self.excludeSizes = [] else: self.excludeSizes = [] if options.excludeTexts: try: self.excludeTexts = list( oset([ excludeText.strip() if excludeText else None for excludeText in options.excludeTexts.split(",") ])) except ValueError: self.excludeTexts = [] else: self.excludeTexts = [] if options.excludeRegexps: try: self.excludeRegexps = list( oset([ excludeRegexp.strip() if excludeRegexp else None for excludeRegexp in options.excludeRegexps.split(",") ])) except ValueError: self.excludeRegexps = [] else: self.excludeRegexps = [] if options.excludeRedirects: try: self.excludeRedirects = list( oset([ excludeRedirect.strip() if excludeRedirect else None for excludeRedirect in options.excludeRedirects.split( ",") ])) except ValueError: self.excludeRedirects = [] else: self.excludeRedirects = [] self.prefixes = [] if not options.prefixes else list( oset([prefix.strip() for prefix in options.prefixes.split(",")])) self.suffixes = [] if not options.suffixes else list( oset([suffix.strip() for suffix in options.suffixes.split(",")])) if options.wordlist: self.wordlist = list( oset([ wordlist.strip() for wordlist in options.wordlist.split(",") ])) else: print("No wordlist was provided, try using -w <wordlist>") exit(1) self.lowercase = options.lowercase self.uppercase = options.uppercase self.capitalization = options.capitalization self.forceExtensions = options.forceExtensions self.data = options.data self.excludeContent = options.excludeContent self.color = options.color self.delay = options.delay self.timeout = options.timeout self.ip = options.ip self.maxRetries = options.maxRetries self.recursive = options.recursive self.deep_recursive = options.deep_recursive self.force_recursive = options.force_recursive self.minimumResponseSize = options.minimumResponseSize self.maximumResponseSize = options.maximumResponseSize self.noExtension = options.noExtension self.onlySelected = options.onlySelected self.outputFile = options.outputFile self.outputFormat = options.outputFormat if options.scanSubdirs: self.scanSubdirs = list( oset([ subdir.strip(" /") + "/" for subdir in options.scanSubdirs.split(",") ])) else: self.scanSubdirs = [] if options.excludeSubdirs: self.excludeSubdirs = list( oset([ subdir.strip(" /") + "/" for subdir in options.excludeSubdirs.split(",") ])) else: self.excludeSubdirs = None if options.skip_on_status: try: self.skip_on_status = list( set([ int(status) for status in options.skip_on_status.split(",") ])) except Exception: print("Invalid skip status code(s)") exit(1) else: self.skip_on_status = [] if options.auth and options.auth_type and (options.auth_type not in [ "basic", "digest", "bearer", "ntlm" ]): print( "'{0}' is not in available authentication types: basic, digest, bearer, ntlm" .format(options.auth_type)) exit(1) elif options.auth and not options.auth_type: print("Please select the authentication type with --auth-type") exit(1) elif options.auth_type and not options.auth: print("No authetication credential found") exit(1) if len(set(self.extensions).intersection(self.excludeExtensions)): print( "Exclude extension list can not contain any extension that has already in the extension list" ) exit(1) self.auth_type = options.auth_type self.auth = options.auth self.redirect = options.followRedirects self.httpmethod = options.httpmethod self.scheme = options.scheme self.requestByHostname = options.requestByHostname self.exit_on_error = options.exit_on_error self.maxrate = options.maxrate self.maxtime = options.maxtime self.recursion_depth = options.recursion_depth if self.scheme not in ["http", "https"]: print("Invalid URI scheme: {0}".format(self.scheme)) exit(1) if self.outputFormat and self.outputFormat not in [ "simple", "plain", "json", "xml", "md", "csv", "html" ]: print( "Select one of the following output formats: simple, plain, json, xml, md, csv, html" ) exit(1)
def __init__(self, script_path): self.script_path = script_path self.parseConfig() options = self.parseArguments() self.quiet = options.quiet self.full_url = options.full_url if not options.url: if options.urlList: with File(options.urlList) as urlList: if not urlList.exists(): print("The file with URLs does not exist") exit(0) if not urlList.is_valid(): print("The file with URLs is invalid") exit(0) if not urlList.can_read(): print("The file with URLs cannot be read") exit(0) self.urlList = list(urlList.get_lines()) elif options.cidr: self.urlList = [str(ip) for ip in IPv4Network(options.cidr)] else: print("URL target is missing, try using -u <url> ") exit(0) else: self.urlList = [options.url] if not options.extensions and not options.noExtension: print( 'WARNING: No extension specified. You need to specify at least one extension.' ) if options.noExtension: options.extensions = str() # Enable to use multiple dictionaries at once for dictFile in options.wordlist.split(','): with File(dictFile) as wordlist: if not wordlist.exists(): print('The wordlist file does not exist') exit(1) if not wordlist.is_valid(): print('The wordlist is invalid') exit(1) if not wordlist.can_read(): print('The wordlist cannot be read') exit(1) if options.proxyList: with File(options.proxyList) as plist: if not plist.exists(): print('The proxylist file does not exist') exit(1) if not plist.is_valid(): print('The proxylist is invalid') exit(1) if not plist.can_read(): print('The proxylist cannot be read') exit(1) self.proxylist = open(options.proxyList).read().splitlines() options.requestByHostname = True elif options.proxy: if options.proxy.startswith( ("http://", "https://", "socks5://", "socks5h://", "socks4://", "socks4a://")): self.proxy = options.proxy else: self.proxy = "http://" + options.proxy options.requestByHostname = True else: self.proxy = None if options.matches_proxy: if options.matches_proxy.startswith( ("http://", "https://", "socks5://", "socks5h://", "socks4://", "socks4a://")): self.matches_proxy = options.matches_proxy else: self.matches_proxy = "http://" + options.matches_proxy else: self.matches_proxy = None if options.headers: try: self.headers = dict( (key, value) for (key, value) in (header.split(":", 1) for header in options.headers)) except Exception: print("Invalid headers") exit(0) else: self.headers = {} if options.headerList: try: with File(options.headerList) as hlist: if not hlist.exists(): print('The header list file does not exist') exit(1) if not hlist.is_valid(): print('The header list file is invalid') exit(1) if not hlist.can_read(): print('The header list cannot be read') exit(1) lines = hlist.get_lines() for line in lines: key, value = line.split(":")[0], line.split(":")[1] self.headers[key] = value except Exception as e: print("Error in headers file: " + str(e)) exit(0) if options.extensions == "*": self.extensions = [ "php", "inc.php", "jsp", "jsf", "asp", "aspx", "do", "action", "cgi", "pl", "html", "htm", "js", "css", "json", "txt", "tar.gz", "tgz" ] else: self.extensions = list( oset([ extension.strip() for extension in options.extensions.split(",") ])) self.useragent = options.useragent self.useRandomAgents = options.useRandomAgents self.cookie = options.cookie if options.threadsCount < 1: print('Threads number must be greater than zero') exit(1) self.threadsCount = options.threadsCount self.includeStatusCodes = [] if options.includeStatusCodes: for statusCode in options.includeStatusCodes.split(","): try: if "-" in statusCode: statusCodes = [ i for i in range( int(statusCode.split("-")[0].strip()), int(statusCode.split("-")[1].strip()) + 1) ] self.includeStatusCodes.extend(statusCodes) else: self.includeStatusCodes.append(int(statusCode.strip())) except ValueError: print( "Invalid status code or status code range: {}".format( statusCode)) exit(0) self.excludeStatusCodes = [] if options.excludeStatusCodes: for statusCode in options.excludeStatusCodes.split(","): try: if "-" in statusCode: statusCodes = [ i for i in range( int(statusCode.split("-")[0].strip()), int(statusCode.split("-")[1].strip()) + 1) ] self.excludeStatusCodes.extend(statusCodes) else: self.excludeStatusCodes.append(int(statusCode.strip())) except ValueError: print( "Invalid status code or status code range: {}".format( statusCode)) exit(0) if options.excludeExtensions: try: self.excludeExtensions = list( oset([ excludeExtension.strip() if excludeExtension else None for excludeExtension in options.excludeExtensions.split(",") ])) except ValueError: self.excludeExtensions = [] else: self.excludeExtensions = [] if options.excludeSizes: try: self.excludeSizes = list( oset([ excludeSize.strip().upper() if excludeSize else None for excludeSize in options.excludeSizes.split(",") ])) except ValueError: self.excludeSizes = [] else: self.excludeSizes = [] if options.excludeTexts: try: self.excludeTexts = list( oset([ excludeText.strip() if excludeText else None for excludeText in options.excludeTexts.split(",") ])) except ValueError: self.excludeTexts = [] else: self.excludeTexts = [] if options.excludeRegexps: try: self.excludeRegexps = list( oset([ excludeRegexp.strip() if excludeRegexp else None for excludeRegexp in options.excludeRegexps.split(",") ])) except ValueError: self.excludeRegexps = [] else: self.excludeRegexps = [] self.prefixes = [] if not options.prefixes else list( oset([prefix.strip() for prefix in options.prefixes.split(',')])) self.suffixes = [] if not options.suffixes else list( oset([suffix.strip() for suffix in options.suffixes.split(',')])) self.wordlist = list( oset([ wordlist.strip() for wordlist in options.wordlist.split(',') ])) self.lowercase = options.lowercase self.uppercase = options.uppercase self.capitalization = options.capitalization self.forceExtensions = options.forceExtensions self.data = options.data self.testFailPath = options.testFailPath self.color = options.color self.delay = options.delay self.timeout = options.timeout self.ip = options.ip self.maxRetries = options.maxRetries self.recursive = options.recursive self.minimumResponseSize = options.minimumResponseSize self.maximumResponseSize = options.maximumResponseSize self.noExtension = options.noExtension self.onlySelected = options.onlySelected self.simpleOutputFile = options.simpleOutputFile self.plainTextOutputFile = options.plainTextOutputFile self.jsonOutputFile = options.jsonOutputFile self.xmlOutputFile = options.xmlOutputFile self.markdownOutputFile = options.markdownOutputFile self.csvOutputFile = options.csvOutputFile if options.scanSubdirs: self.scanSubdirs = list( oset([ subdir.strip(" /") + "/" for subdir in options.scanSubdirs.split(",") ])) else: self.scanSubdirs = [] if not self.recursive and options.excludeSubdirs: self.excludeSubdirs = None elif options.excludeSubdirs: self.excludeSubdirs = list( oset([ subdir.strip() for subdir in options.excludeSubdirs.split(",") ])) for i in range(len(self.excludeSubdirs)): while self.excludeSubdirs[i].startswith("/"): self.excludeSubdirs[i] = self.excludeSubdirs[i][1:] while self.excludeSubdirs[i].endswith("/"): self.excludeSubdirs[i] = self.excludeSubdirs[i][:-1] self.excludeSubdirs = list(oset(self.excludeSubdirs)) else: self.excludeSubdirs = None if len(set(self.extensions).intersection(self.excludeExtensions)): print( "Exclude extension list can not contain any extension that has already in the extension list" ) exit(0) self.redirect = options.followRedirects self.httpmethod = options.httpmethod self.requestByHostname = options.requestByHostname self.exit_on_error = options.exit_on_error self.debug = options.debug self.recursive_level_max = options.recursive_level_max
def __init__(self, script_path): self.script_path = script_path self.parseConfig() options = self.parseArguments() self.quiet = options.quiet self.full_url = options.full_url if not options.url: if options.urlList: with File(options.urlList) as urlList: if not urlList.exists(): print("The file with URLs does not exist") exit(0) if not urlList.is_valid(): print("The file with URLs is invalid") exit(0) if not urlList.can_read(): print("The file with URLs cannot be read") exit(0) self.urlList = list(urlList.get_lines()) elif not options.url: print("URL target is missing, try using -u <url> ") exit(0) else: self.urlList = [options.url] if not options.extensions and not options.defaultExtensions and not options.noExtension: print( 'No extension specified. You must specify at least one extension or try using default extension list.' ) exit(0) if not options.extensions and options.defaultExtensions: options.extensions = self.defaultExtensions if options.noExtension: options.extensions = "" # Enable to use multiple dictionaries at once for dictFile in options.wordlist.split(','): with File(dictFile) as wordlist: if not wordlist.exists(): print('The wordlist file does not exist') exit(1) if not wordlist.is_valid(): print('The wordlist is invalid') exit(1) if not wordlist.can_read(): print('The wordlist cannot be read') exit(1) if options.proxyList: with File(options.proxyList) as plist: if not plist.exists(): print('The proxylist file does not exist') exit(1) if not plist.is_valid(): print('The proxylist is invalid') exit(1) if not plist.can_read(): print('The proxylist cannot be read') exit(1) self.proxylist = open(options.proxyList).read().splitlines() elif options.httpProxy: if options.httpProxy.startswith( "http://") or options.httpProxy.startswith( "https://") or options.httpProxy.startswith( "socks5://"): self.proxy = options.httpProxy else: self.proxy = "http://{0}".format(options.httpProxy) else: self.proxy = None if options.headers: try: self.headers = dict( (key, value) for (key, value) in (header.split(":", 1) for header in options.headers)) except Exception: print("Invalid headers") exit(0) else: self.headers = {} if options.headerList: try: with File(options.headerList) as hlist: if not hlist.exists(): print('The header list file does not exist') exit(1) if not hlist.is_valid(): print('The header list file is invalid') exit(1) if not hlist.can_read(): print('The header list cannot be read') exit(1) lines = hlist.get_lines() for line in lines: key, value = line.split(":")[0], line.split(":")[1] self.headers[key] = value except Exception as e: print("Error in headers file: " + str(e)) exit(0) self.extensions = list( oset([ extension.strip() for extension in options.extensions.split(",") ])) self.useragent = options.useragent self.useRandomAgents = options.useRandomAgents self.cookie = options.cookie if options.threadsCount < 1: print('Threads number must be greater than zero') exit(1) self.threadsCount = options.threadsCount if options.includeStatusCodes: try: self.includeStatusCodes = list( oset([ int(includeStatusCode.strip()) if includeStatusCode else None for includeStatusCode in options.includeStatusCodes.split(',') ])) except ValueError: self.includeStatusCodes = [] else: self.includeStatusCodes = [] if options.excludeExtensions: try: self.excludeExtensions = list( oset([ excludeExtension.strip() if excludeExtension else None for excludeExtension in options.excludeExtensions.split(",") ])) except ValueError: self.excludeExtensions = [] else: self.excludeExtensions = [] if options.excludeStatusCodes: try: self.excludeStatusCodes = list( oset([ int(excludeStatusCode.strip()) if excludeStatusCode else None for excludeStatusCode in options.excludeStatusCodes.split(",") ])) except ValueError: self.excludeStatusCodes = [] else: self.excludeStatusCodes = [] if options.excludeSizes: try: self.excludeSizes = list( oset([ excludeSize.strip().upper() if excludeSize else None for excludeSize in options.excludeSizes.split(",") ])) except ValueError: self.excludeSizes = [] else: self.excludeSizes = [] if options.excludeTexts: try: self.excludeTexts = list( oset([ excludeText.strip() if excludeText else None for excludeText in options.excludeTexts.split(",") ])) except ValueError: self.excludeTexts = [] else: self.excludeTexts = [] if options.excludeRegexps: try: self.excludeRegexps = list( oset([ excludeRegexp.strip() if excludeRegexp else None for excludeRegexp in options.excludeRegexps.split(",") ])) except ValueError: self.excludeRegexps = [] else: self.excludeRegexps = [] self.prefixes = [] if not options.prefixes else list( oset([prefix.strip() for prefix in options.prefixes.split(',')])) self.suffixes = [] if not options.suffixes else list( oset([suffix.strip() for suffix in options.suffixes.split(',')])) self.wordlist = list( oset([ wordlist.strip() for wordlist in options.wordlist.split(',') ])) self.lowercase = options.lowercase self.uppercase = options.uppercase self.capitalization = options.capitalization self.forceExtensions = options.forceExtensions self.data = options.data self.noDotExtensions = options.noDotExtensions self.simpleOutputFile = options.simpleOutputFile self.plainTextOutputFile = options.plainTextOutputFile self.jsonOutputFile = options.jsonOutputFile self.xmlOutputFile = options.xmlOutputFile self.delay = options.delay self.timeout = options.timeout self.ip = options.ip self.maxRetries = options.maxRetries self.recursive = options.recursive self.suppressEmpty = options.suppressEmpty self.minimumResponseSize = options.minimumResponseSize self.maximumResponseSize = options.maximumResponseSize self.noExtension = options.noExtension if options.scanSubdirs: self.scanSubdirs = list( oset([ subdir.strip() for subdir in options.scanSubdirs.split(",") ])) for i in range(len(self.scanSubdirs)): while self.scanSubdirs[i].startswith("/"): self.scanSubdirs[i] = self.scanSubdirs[i][1:] while self.scanSubdirs[i].endswith("/"): self.scanSubdirs[i] = self.scanSubdirs[i][:-1] self.scanSubdirs = list( oset([subdir + "/" for subdir in self.scanSubdirs])) else: self.scanSubdirs = None if not self.recursive and options.excludeSubdirs: self.excludeSubdirs = None elif options.excludeSubdirs: self.excludeSubdirs = list( oset([ subdir.strip() for subdir in options.excludeSubdirs.split(",") ])) for i in range(len(self.excludeSubdirs)): while self.excludeSubdirs[i].startswith("/"): self.excludeSubdirs[i] = self.excludeSubdirs[i][1:] while self.excludeSubdirs[i].endswith("/"): self.excludeSubdirs[i] = self.excludeSubdirs[i][:-1] self.excludeSubdirs = list(oset(self.excludeSubdirs)) else: self.excludeSubdirs = None if len(set(self.extensions).intersection(self.excludeExtensions)): print( "Exclude extensions can not contain any extension that has already in the extensions" ) exit(0) self.redirect = options.noFollowRedirects self.requestByHostname = options.requestByHostname self.stop = options.stop self.httpmethod = options.httpmethod self.recursive_level_max = options.recursive_level_max