def parse(self):
# Parse for 2 situations: \n as a newline or \r\n as a newline
self.parsed = self.raw_content.split("\n\n")
if len(self.parsed) == 1:
self.parsed = self.raw_content.split("\r\n\r\n")
self.startline = self.parsed[0].splitlines()[0]
try:
self.headers_parser = HeadersParser(
self.parsed[0].splitlines()[1:])
except Exception:
print("Invalid headers in the raw request")
exit(1)
try:
self.body = self.parsed[1] if self.parsed[1] else None
except IndexError:
self.body = None
try:
self.host = self.headers_parser.lower_headers["host"].strip()
except KeyError:
print("Can't find the Host header in the raw request")
exit(1)
self.path = self.startline.split(" ")[1]
def parse(self):
self.head = self.raw_content.split(NEW_LINE * 2)[0].splitlines(0)
try:
self.headers_ = HeadersParser(self.head[1:])
except Exception:
print("Invalid headers in the raw request")
exit(1)
try:
self.body = self.raw_content.split(NEW_LINE * 2)[1]
except IndexError:
self.body = None
try:
self.host = self.headers_.get("host").strip()
except KeyError:
print("Can't find the Host header in the raw request")
exit(1)
self.method, self.path = self.head[0].split()[:2]
def parse_raw(raw_file):
with File(raw_file) as content:
raw_content = content.read()
head = raw_content.split(NEW_LINE * 2)[0].splitlines(0)
method, path = head[0].split()[:2]
try:
headers = HeadersParser(head[1:])
host = headers.get("host").strip()
except KeyError:
print("Can't find the Host header in the raw request")
exit(1)
except Exception:
print("Invalid headers in the raw request")
exit(1)
try:
body = raw_content.split(NEW_LINE * 2)[1]
except IndexError:
body = None
return [host + path], method, dict(headers), body
class Raw(object):
def __init__(self, raw_file):
with File(raw_file) as raw_content:
self.raw_content = raw_content.read()
self.parse()
def parse(self):
self.head = self.raw_content.split(NEW_LINE * 2)[0].splitlines(0)
try:
self.headers_ = HeadersParser(self.head[1:])
except Exception:
print("Invalid headers in the raw request")
exit(1)
try:
self.body = self.raw_content.split(NEW_LINE * 2)[1]
except IndexError:
self.body = None
try:
self.host = self.headers_.get("host").strip()
except KeyError:
print("Can't find the Host header in the raw request")
exit(1)
self.method, self.path = self.head[0].split()[:2]
@property
def url(self):
return "{0}{1}".format(self.host, self.path)
@property
def headers(self):
return dict(self.headers_)
def __init__(self, script_path):
self.script_path = script_path
self.parse_config()
options = self.parse_arguments()
self.quiet = options.quiet
self.full_url = options.full_url
self.url_list = None
self.raw_file = None
if not options.url:
if options.url_list:
file = self.access_file(options.url_list, "file contains URLs")
self.url_list = list(file.get_lines())
elif options.cidr:
self.url_list = iprange(options.cidr)
elif options.stdin_urls:
self.url_list = sys.stdin.read().splitlines()
elif options.raw_file:
self.access_file(options.raw_file, "file with raw request")
self.raw_file = options.raw_file
else:
print("URL target is missing, try using -u <url>")
exit(1)
else:
self.url_list = [options.url]
self.url_list = uniq(self.url_list)
if not options.extensions and not options.no_extension:
print("WARNING: No extension was specified!")
if options.no_extension:
options.extensions = str()
for dict_file in options.wordlist.split(","):
self.access_file(dict_file, "wordlist")
if options.proxy_list:
file = self.access_file(options.proxy_list, "proxylist file")
self.proxylist = file.read().splitlines()
options.request_by_hostname = True
elif options.proxy:
self.proxy = options.proxy
options.request_by_hostname = True
else:
self.proxy = None
if options.replay_proxy:
self.replay_proxy = options.replay_proxy
options.request_by_hostname = True
else:
self.replay_proxy = None
self.headers = {}
if options.header_list:
try:
file = self.access_file(options.header_list,
"header list file")
self.headers.update(HeadersParser(file.read()).headers)
except Exception as e:
print("Error in headers file: " + str(e))
exit(1)
if options.headers:
try:
self.headers.update(HeadersParser(options.headers).headers)
except Exception:
print("Invalid headers")
exit(1)
if options.extensions == "*":
self.extensions = [
"php", "jsp", "asp", "aspx", "do", "action", "cgi", "pl",
"html", "htm", "js", "json", "tar.gz", "bak"
]
elif options.extensions == "banner.txt":
print(
"A weird extension was provided: 'banner.txt'. Please do not use * as the extension or enclose it in double quotes"
)
exit(0)
else:
self.extensions = uniq([
extension.lstrip(' .')
for extension in options.extensions.split(",")
])
if options.exclude_extensions:
self.exclude_extensions = uniq([
exclude_extension.lstrip(' .')
for exclude_extension in options.exclude_extensions.split(",")
])
else:
self.exclude_extensions = []
self.useragent = options.useragent
self.use_random_agents = options.use_random_agents
self.cookie = options.cookie
if options.threads_count < 1:
print("Threads number must be greater than zero")
exit(1)
self.threads_count = options.threads_count
if options.include_status_codes:
self.include_status_codes = self.parse_status_codes(
options.include_status_codes)
else:
self.include_status_codes = []
if options.exclude_status_codes:
self.exclude_status_codes = self.parse_status_codes(
options.exclude_status_codes)
else:
self.exclude_status_codes = []
if options.recursion_status_codes:
self.recursion_status_codes = self.parse_status_codes(
options.recursion_status_codes)
else:
self.recursion_status_codes = []
if options.exclude_sizes:
try:
self.exclude_sizes = uniq([
exclude_size.strip().upper() if exclude_size else None
for exclude_size in options.exclude_sizes.split(",")
])
except ValueError:
self.exclude_sizes = []
else:
self.exclude_sizes = []
if options.exclude_texts:
try:
self.exclude_texts = uniq([
exclude_text.strip() if exclude_text else None
for exclude_text in options.exclude_texts.split(",")
])
except ValueError:
self.exclude_texts = []
else:
self.exclude_texts = []
if options.exclude_regexps:
try:
self.exclude_regexps = uniq([
exclude_regexp.strip() if exclude_regexp else None
for exclude_regexp in options.exclude_regexps.split(",")
])
except ValueError:
self.exclude_regexps = []
else:
self.exclude_regexps = []
if options.exclude_redirects:
try:
self.exclude_redirects = uniq([
exclude_redirect.strip() if exclude_redirect else None for
exclude_redirect in options.exclude_redirects.split(",")
])
except ValueError:
self.exclude_redirects = []
else:
self.exclude_redirects = []
self.prefixes = uniq([
prefix.strip() for prefix in options.prefixes.split(",")
]) if options.prefixes else []
self.suffixes = uniq([
suffix.strip() for suffix in options.suffixes.split(",")
]) if options.suffixes else []
if options.wordlist:
self.wordlist = uniq(
[wordlist.strip() for wordlist in options.wordlist.split(",")])
else:
print("No wordlist was provided, try using -w <wordlist>")
exit(1)
self.lowercase = options.lowercase
self.uppercase = options.uppercase
self.capitalization = options.capitalization
self.force_extensions = options.force_extensions
self.data = options.data
self.exclude_content = options.exclude_content
self.color = options.color
self.delay = options.delay
self.timeout = options.timeout
self.ip = options.ip
self.max_retries = options.max_retries
self.recursive = options.recursive
self.deep_recursive = options.deep_recursive
self.force_recursive = options.force_recursive
self.minimum_response_size = options.minimum_response_size
self.maximum_response_size = options.maximum_response_size
self.no_extension = options.no_extension
self.only_selected = options.only_selected
self.output_file = options.output_file
self.output_format = options.output_format
self.scan_subdirs = []
if options.scan_subdirs:
for subdir in options.scan_subdirs.split(","):
subdir = subdir.strip(" ")
if subdir.startswith("/"):
subdir = subdir[1:]
self.scan_subdirs.append(subdir)
self.exclude_subdirs = []
if options.exclude_subdirs:
for subdir in options.exclude_subdirs.split(","):
subdir = subdir.strip(" ")
if subdir.startswith("/"):
subdir = subdir[1:]
self.exclude_subdirs.append(subdir)
if options.skip_on_status:
self.skip_on_status = self.parse_status_codes(
options.skip_on_status)
else:
self.skip_on_status = []
if options.auth and options.auth_type and (options.auth_type not in [
"basic", "digest", "bearer", "ntlm"
]):
print(
"'{0}' is not in available authentication types: basic, digest, bearer, ntlm"
.format(options.auth_type))
exit(1)
elif options.auth and not options.auth_type:
print("Please select the authentication type with --auth-type")
exit(1)
elif options.auth_type and not options.auth:
print("No authentication credential found")
exit(1)
if len(set(self.extensions).intersection(self.exclude_extensions)):
print(
"Exclude extension list can not contain any extension that has already in the extension list"
)
exit(1)
self.auth_type = options.auth_type
self.auth = options.auth
self.redirect = options.follow_redirects
self.httpmethod = options.httpmethod
self.scheme = options.scheme
self.request_by_hostname = options.request_by_hostname
self.exit_on_error = options.exit_on_error
self.maxrate = options.maxrate
self.maxtime = options.maxtime
self.recursion_depth = options.recursion_depth
if self.scheme not in ["http", "https"]:
print("Invalid URI scheme: {0}".format(self.scheme))
exit(1)
if self.output_format and self.output_format not in [
"simple", "plain", "json", "xml", "md", "csv", "html"
]:
print(
"Select one of the following output formats: simple, plain, json, xml, md, csv, html"
)
exit(1)
def options():
opt = parse_config(parse_arguments())
if opt.session_file:
return AttributeDict(vars(opt))
opt.httpmethod = opt.httpmethod.upper()
if opt.url_file:
fd = access_file(opt.url_file, "file contains URLs")
opt.urls = fd.get_lines()
elif opt.cidr:
opt.urls = iprange(opt.cidr)
elif opt.stdin_urls:
opt.urls = sys.stdin.read().splitlines(0)
opt.urls = uniq(opt.urls)
if opt.raw_file:
access_file(opt.raw_file, "file with raw request")
elif not opt.urls:
print("URL target is missing, try using -u <url>")
exit(1)
if not opt.extensions and not opt.no_extension:
print("WARNING: No extension was specified!")
for dict_file in opt.wordlist.split(","):
access_file(dict_file, "wordlist")
if opt.threads_count < 1:
print("Threads number must be greater than zero")
exit(1)
if opt.tor:
opt.proxy = [DEFAULT_TOR_PROXIES]
elif opt.proxy_file:
fd = access_file(opt.proxy_file, "proxy list file")
opt.proxy = fd.get_lines()
if opt.data_file:
fd = access_file(opt.data_file, "data file")
opt.data = fd.get_lines()
headers = {}
if opt.header_file:
try:
fd = access_file(opt.header_file, "header list file")
headers.update(dict(HeadersParser(fd.read())))
except Exception as e:
print("Error in headers file: " + str(e))
exit(1)
if opt.headers:
try:
headers.update(dict(HeadersParser("\n".join(opt.headers))))
except Exception:
print("Invalid headers")
exit(1)
opt.headers = headers
opt.include_status_codes = parse_status_codes(opt.include_status_codes)
opt.exclude_status_codes = parse_status_codes(opt.exclude_status_codes)
opt.recursion_status_codes = parse_status_codes(opt.recursion_status_codes)
opt.skip_on_status = parse_status_codes(opt.skip_on_status)
opt.prefixes = set(prefix.strip() for prefix in opt.prefixes.split(","))
opt.suffixes = set(suffix.strip() for suffix in opt.suffixes.split(","))
opt.exclude_extensions = uniq([
exclude_extension.lstrip(" .")
for exclude_extension in opt.exclude_extensions.split(",")
])
opt.exclude_sizes = uniq([
exclude_size.strip().upper()
for exclude_size in opt.exclude_sizes.split(",")
])
opt.exclude_texts = uniq([
exclude_text.strip() for exclude_text in opt.exclude_texts.split(",")
])
opt.scan_subdirs = [
subdir.lstrip(" /") +
("" if not subdir or subdir.endswith("/") else "/")
for subdir in opt.scan_subdirs.split(",")
]
opt.exclude_subdirs = [
subdir.lstrip(" /") +
("" if not subdir or subdir.endswith("/") else "/")
for subdir in opt.exclude_subdirs.split(",")
]
if opt.no_extension:
opt.extensions = ""
elif opt.extensions == "*":
opt.extensions = COMMON_EXTENSIONS
elif opt.extensions == "CHANGELOG.md":
print(
"A weird extension was provided: 'CHANGELOG.md'. Please do not use * as the "
"extension or enclose it in double quotes")
exit(0)
else:
opt.extensions = uniq([
extension.lstrip(" .") for extension in opt.extensions.split(",")
])
if not opt.wordlist:
print("No wordlist was provided, try using -w <wordlist>")
exit(1)
opt.wordlist = uniq(
[wordlist.strip() for wordlist in opt.wordlist.split(",")])
if opt.auth and not opt.auth_type:
print("Please select the authentication type with --auth-type")
exit(1)
elif opt.auth_type and not opt.auth:
print("No authentication credential found")
exit(1)
elif opt.auth and opt.auth_type not in AUTHENTICATION_TYPES:
print(f"'{opt.auth_type}' is not in available authentication "
f"types: {', '.join(AUTHENTICATION_TYPES)}")
exit(1)
if set(opt.extensions).intersection(opt.exclude_extensions):
print("Exclude extension list can not contain any extension "
"that has already in the extension list")
exit(1)
if opt.output_format not in OUTPUT_FORMATS:
print("Select one of the following output formats: "
f"{', '.join(OUTPUT_FORMATS)}")
exit(1)
return AttributeDict(vars(opt))
def __init__(self):
options = self.parse_config(arguments_parser())
self.__dict__.update(options.__dict__)
self.httpmethod = self.httpmethod.upper()
self.url_list = []
if options.url:
self.url_list = [options.url]
elif options.url_list:
file = self.access_file(options.url_list, "file contains URLs")
self.url_list = file.get_lines()
elif options.cidr:
self.url_list = iprange(options.cidr)
elif options.stdin_urls:
self.url_list = sys.stdin.read().splitlines(0)
if options.raw_file:
self.access_file(options.raw_file, "file with raw request")
elif not len(self.url_list):
print("URL target is missing, try using -u <url>")
exit(1)
self.url_list = uniq(self.url_list)
if not options.extensions and not options.no_extension:
print("WARNING: No extension was specified!")
for dict_file in options.wordlist.split(","):
self.access_file(dict_file, "wordlist")
if options.threads_count < 1:
print("Threads number must be greater than zero")
exit(1)
if options.proxy_list:
file = self.access_file(options.proxy_list, "proxylist file")
self.proxylist = file.get_lines()
if self.proxy or self.proxylist or self.replay_proxy:
self.request_by_hostname = True
self.headers = {}
if options.header_list:
try:
file = self.access_file(options.header_list,
"header list file")
self.headers.update(HeadersParser(file.read()).headers)
except Exception as e:
print("Error in headers file: " + str(e))
exit(1)
if options.headers:
try:
self.headers.update(HeadersParser(options.headers).headers)
except Exception:
print("Invalid headers")
exit(1)
if options.extensions == "*":
self.extensions = COMMON_EXTENSIONS
elif options.extensions == "banner.txt":
print(
"A weird extension was provided: 'banner.txt'. Please do not use * as the extension or enclose it in double quotes"
)
exit(0)
if options.no_extension:
self.extensions = ""
self.include_status_codes = self.parse_status_codes(
options.include_status_codes)
self.exclude_status_codes = self.parse_status_codes(
options.exclude_status_codes)
self.recursion_status_codes = self.parse_status_codes(
options.recursion_status_codes)
self.skip_on_status = self.parse_status_codes(options.skip_on_status)
self.prefixes = uniq(
[prefix.strip() for prefix in self.prefixes.split(",")])
self.suffixes = uniq(
[suffix.strip() for suffix in self.suffixes.split(",")])
self.extensions = uniq([
extension.lstrip(' .')
for extension in options.extensions.split(",")
])
self.exclude_extensions = uniq([
exclude_extension.lstrip(' .')
for exclude_extension in options.exclude_extensions.split(",")
])
self.exclude_sizes = uniq([
exclude_size.strip().upper()
for exclude_size in options.exclude_sizes.split(",")
])
self.exclude_texts = uniq([
exclude_text.strip()
for exclude_text in options.exclude_texts.split(",")
])
self.exclude_regexps = uniq([
exclude_regexp.strip()
for exclude_regexp in options.exclude_regexps.split(",")
])
self.exclude_redirects = uniq([
exclude_redirect.strip()
for exclude_redirect in options.exclude_redirects.split(",")
])
self.scan_subdirs = [
subdir.strip().ljust(len(subdir) + 1, "/").lstrip("/")
for subdir in options.scan_subdirs.split(",")
]
self.exclude_subdirs = [
subdir.strip().ljust(len(subdir) + 1, "/").lstrip("/")
for subdir in options.exclude_subdirs.split(",")
]
if not options.wordlist:
print("No wordlist was provided, try using -w <wordlist>")
exit(1)
self.wordlist = uniq(
[wordlist.strip() for wordlist in options.wordlist.split(",")])
if options.auth and not options.auth_type:
print("Please select the authentication type with --auth-type")
exit(1)
elif options.auth_type and not options.auth:
print("No authentication credential found")
exit(1)
elif options.auth and options.auth_type not in AUTHENTICATION_TYPES:
print("'{}' is not in available authentication types: {}".format(
options.auth_type, ", ".join(AUTHENTICATION_TYPES)))
exit(1)
if len(set(self.extensions).intersection(self.exclude_extensions)):
print(
"Exclude extension list can not contain any extension that has already in the extension list"
)
exit(1)
if self.output_format and self.output_format not in list(
OUTPUT_FORMATS):
print("Select one of the following output formats: {}".format(
", ".join(OUTPUT_FORMATS)))
exit(1)