async def _async_try_login_post(
self, username: str, password: str,
auth_url: str) -> Tuple[bool, dict, List[str]]:
# Fetch the login page and try to extract the login form
try:
page = await self.async_get(web.Request(auth_url),
follow_redirects=True)
form = {}
disconnect_urls = []
login_form, username_field_idx, password_field_idx = page.find_login_form(
)
if login_form:
post_params = login_form.post_params
get_params = login_form.get_params
if login_form.method == "POST":
post_params[username_field_idx][1] = username
post_params[password_field_idx][1] = password
form["login_field"] = post_params[username_field_idx][0]
form["password_field"] = post_params[password_field_idx][0]
else:
get_params[username_field_idx][1] = username
get_params[password_field_idx][1] = password
form["login_field"] = get_params[username_field_idx][0]
form["password_field"] = get_params[password_field_idx][0]
login_request = web.Request(path=login_form.url,
method=login_form.method,
post_params=post_params,
get_params=get_params,
referer=login_form.referer,
link_depth=login_form.link_depth)
login_response = await self.async_send(login_request,
follow_redirects=True)
# ensure logged in
if login_response.soup.find_all(
text=re.compile(DISCONNECT_REGEX)):
self.is_logged_in = True
logging.success(_("Login success"))
disconnect_urls = self._extract_disconnect_urls(
login_response)
else:
logging.warning(
_("Login failed") + " : " +
_("Credentials might be invalid"))
else:
logging.warning(
_("Login failed") + " : " + _("No login form detected"))
return self.is_logged_in, form, disconnect_urls
except ConnectionError:
logging.error(_("[!] Connection error with URL"), auth_url)
except httpx.RequestError as error:
logging.error(
_("[!] {} with url {}").format(error.__class__.__name__,
auth_url))
async def async_try_login(self, auth_url: str):
"""Try to authenticate with the provided url and credentials."""
if len(self._auth_credentials) != 2:
logging.error(
_("Login failed") + " : " + _("Invalid credentials format"))
return
username, password = self._auth_credentials
# Fetch the login page and try to extract the login form
try:
page = await self.async_get(web.Request(auth_url),
follow_redirects=True)
login_form, username_field_idx, password_field_idx = page.find_login_form(
)
if login_form:
post_params = login_form.post_params
get_params = login_form.get_params
if login_form.method == "POST":
post_params[username_field_idx][1] = username
post_params[password_field_idx][1] = password
else:
get_params[username_field_idx][1] = username
get_params[password_field_idx][1] = password
login_request = web.Request(path=login_form.url,
method=login_form.method,
post_params=post_params,
get_params=get_params,
referer=login_form.referer,
link_depth=login_form.link_depth)
login_response = await self.async_send(login_request,
follow_redirects=True)
# ensure logged in
if login_response.soup.find_all(text=re.compile(
r'(?i)((log|sign)\s?out|disconnect|déconnexion)')):
self.is_logged_in = True
logging.success(_("Login success"))
else:
logging.warning(
_("Login failed") + " : " +
_("Credentials might be invalid"))
else:
logging.warning(
_("Login failed") + " : " + _("No login form detected"))
except ConnectionError:
logging.error(_("[!] Connection error with URL"), auth_url)
except httpx.RequestError as error:
logging.error(
_("[!] {} with url {}").format(error.__class__.__name__,
auth_url))
def __init__(self, crawler, xml_report_generator, logger, attack_options):
Attack.__init__(self, crawler, xml_report_generator, logger, attack_options)
user_config_dir = os.getenv('HOME') or os.getenv('USERPROFILE')
user_config_dir += "/config"
if not os.path.isdir(user_config_dir):
os.makedirs(user_config_dir)
try:
with open(os.path.join(user_config_dir, self.NIKTO_DB)) as fd:
reader = csv.reader(fd)
self.nikto_db = [line for line in reader if line != [] and line[0].isdigit()]
except IOError:
try:
print(_("Problem with local nikto database."))
print(_("Downloading from the web..."))
nikto_req = web.Request("http://cirt.net/nikto/UPDATES/2.1.5/db_tests")
response = self.crawler.send(nikto_req)
csv.register_dialect("nikto", quoting=csv.QUOTE_ALL, doublequote=False, escapechar="\\")
reader = csv.reader(response.content.split("\n"), "nikto")
self.nikto_db = [line for line in reader if line != [] and line[0].isdigit()]
with open(os.path.join(user_config_dir, self.NIKTO_DB), "w") as fd:
writer = csv.writer(fd)
writer.writerows(self.nikto_db)
except socket.timeout:
print(_("Error downloading Nikto database"))
async def _async_try_login_basic_digest_ntlm(
self, auth_url: str) -> Tuple[bool, dict, List[str]]:
page = await self.async_get(web.Request(auth_url))
if page.status in (401, 403, 404):
return False, {}, []
return True, {}, []
def test_credentials(self, login_form, username_index, password_index,
username, password):
post_params = login_form.post_params
get_params = login_form.get_params
if login_form.method == "POST":
post_params[username_index][1] = username
post_params[password_index][1] = password
else:
get_params[username_index][1] = username
get_params[password_index][1] = password
login_request = web.Request(path=login_form.url,
method=login_form.method,
post_params=post_params,
get_params=get_params,
referer=login_form.referer,
link_depth=login_form.link_depth)
try:
login_response = self.crawler.send(login_request,
follow_redirects=True)
except ReadTimeout:
return ""
return login_response.content
def __init__(self,
base_url: str,
timeout: float = 10.0,
secure: bool = False,
compression: bool = True,
proxies: dict = None,
user_agent: str = None):
self._timeout = timeout
self._session = requests.Session()
if user_agent:
self._session.headers["User-Agent"] = user_agent
else:
self._session.headers[
"User-Agent"] = "Mozilla/5.0 (Windows NT 6.1; rv:45.0) Gecko/20100101 Firefox/45.0"
self._session.headers["Accept-Language"] = "en-US"
self._session.headers["Accept-Encoding"] = "gzip, deflate, br"
self._session.headers[
"Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"
self._session.max_redirects = 5
self._session.verify = secure
self._scope = Scope.FOLDER
self._base = web.Request(base_url)
self.auth_url = self._base.url
self.is_logged_in = False
if not compression:
self._session.headers["accept-encoding"] = "identity"
if proxies is not None and isinstance(proxies, dict):
# ex: {'http': 'http://127.0.0.1:8080'}
self._session.proxies = proxies
self._auth_credentials = {}
self._auth_method = "basic"
def attack(self):
http_resources = self.persister.get_links(attack_module=self.name) if self.do_get else []
for original_request in http_resources:
try:
url = original_request.path
if self.verbose == 2:
print("[¨] {0}".format(url))
if url not in self.attacked_get:
self.attacked_get.append(url)
evil_req = web.Request(url)
resp = self.crawler.send(evil_req, headers=self.hdrs)
if resp:
data = resp.content
if self.rand_string in data:
self.log_red(_("URL {0} seems vulnerable to Shellshock attack!").format(url))
self.add_vuln(
request_id=original_request.path_id,
category=Vulnerability.EXEC,
level=Vulnerability.HIGH_LEVEL,
request=evil_req,
info=_("URL {0} seems vulnerable to Shellshock attack").format(url)
)
except (RequestException, KeyboardInterrupt) as exception:
yield exception
yield original_request
def find_login_form(self):
"""Returns the login Request extracted from the Page, the username and password fields."""
for form in self.soup.find_all("form"):
username_field_idx = []
password_field_idx = []
for i, input_field in enumerate(form.find_all("input")):
input_type = input_field.attrs.get("type", "text").lower()
input_name = input_field.attrs.get("name", "undefined").lower()
input_id = input_field.attrs.get("id", "undefined").lower()
if input_type == "email":
username_field_idx.append(i)
elif input_type == "text" and (any(
field_name in input_name
for field_name in ["mail", "user", "login", "name"]
) or any(field_id in input_id
for field_id in ["mail", "user", "login", "name"])):
username_field_idx.append(i)
elif input_type == "password":
password_field_idx.append(i)
# ensure login form
if len(username_field_idx) == 1 and len(password_field_idx) == 1:
inputs = form.find_all("input", attrs={"name": True})
url = self.make_absolute(
form.attrs.get("action", "").strip() or self.url)
method = form.attrs.get("method", "GET").strip().upper()
enctype = form.attrs.get(
"enctype", "application/x-www-form-urlencoded").lower()
post_params = []
get_params = []
if method == "POST":
post_params = [[
input_data["name"],
input_data.get("value", "")
] for input_data in inputs]
else:
get_params = [[
input_data["name"],
input_data.get("value", "")
] for input_data in inputs]
login_form = web.Request(
url,
method=method,
post_params=post_params,
get_params=get_params,
encoding=self.apparent_encoding,
referer=self.url,
enctype=enctype,
)
return login_form, username_field_idx[0], password_field_idx[0]
return None, "", ""
def get_path_by_id(self, path_id):
cursor = self._conn.cursor()
cursor.execute("SELECT * FROM paths WHERE path_id = ? LIMIT 1",
(path_id, ))
row = cursor.fetchone()
if not row:
return None
get_params = []
post_params = []
file_params = []
for param_row in cursor.execute(
("SELECT type, name, value1, value2, meta "
"FROM params "
"WHERE path_id = ? "
"ORDER BY type, param_order"), (path_id, )):
name = param_row[1]
value1 = param_row[2]
if param_row[0] == "GET":
get_params.append([name, value1])
elif param_row[0] == "POST":
if name == "__RAW__" and not post_params:
# First POST parameter is __RAW__, it should mean that we have raw content
post_params = value1
elif isinstance(post_params, list):
post_params.append([name, value1])
elif param_row[0] == "FILE":
if param_row[4]:
file_params.append(
[name, (value1, param_row[3], param_row[4])])
else:
file_params.append([name, (value1, param_row[3])])
else:
raise ValueError("Unknown param type {}".format(param_row[0]))
request = web.Request(row[1],
method=row[2],
encoding=row[5],
enctype=row[3],
referer=row[8],
get_params=get_params,
post_params=post_params,
file_params=file_params)
if row[6]:
request.status = row[6]
if row[7]:
request.set_headers(json.loads(row[7]))
request.link_depth = row[4]
request.path_id = path_id
return request
def attack(self):
http_resources = self.persister.get_links(
attack_module=self.name) if self.do_get else []
for original_request in http_resources:
if original_request.file_name == "":
yield original_request
continue
page = original_request.path
headers = original_request.headers
# Do not attack application-type files
if "content-type" not in headers:
# Sometimes there's no content-type... so we rely on the document extension
if (page.split(".")[-1]
not in self.allowed) and page[-1] != "/":
yield original_request
continue
elif "text" not in headers["content-type"]:
yield original_request
continue
for payload, flags in self.payloads:
try:
payload = payload.replace("[FILE_NAME]",
original_request.file_name)
payload = payload.replace(
"[FILE_NOEXT]",
splitext(original_request.file_name)[0])
url = page.replace(original_request.file_name, payload)
if self.verbose == 2:
print("[¨] {0}".format(url))
if url not in self.attacked_get:
self.attacked_get.append(url)
evil_req = web.Request(url)
response = self.crawler.send(evil_req)
if response and response.status == 200:
self.log_red(
_("Found backup file {}".format(evil_req.url)))
self.add_vuln(
request_id=original_request.path_id,
category=NAME,
level=LOW_LEVEL,
request=evil_req,
info=_("Backup file {0} found for {1}").format(
url, page))
except (KeyboardInterrupt, RequestException) as exception:
yield exception
yield original_request
def test_directory(self, path: str):
if self.verbose == 2:
print("[¨] Testing directory {0}".format(path))
test_page = web.Request(path + "does_n0t_exist.htm")
try:
response = self.crawler.send(test_page)
if response.status not in [403, 404]:
# we don't want to deal with this at the moment
return
for candidate, flags in self.payloads:
url = path + candidate
if url not in self.known_dirs and url not in self.known_pages and url not in self.new_resources:
page = web.Request(path + candidate)
try:
response = self.crawler.send(page)
if response.redirection_url:
loc = response.redirection_url
# if loc in self.known_dirs or loc in self.known_pages:
# continue
if response.is_directory_redirection:
self.log_red("Found webpage {0}", loc)
self.new_resources.append(loc)
else:
self.log_red("Found webpage {0}", page.path)
self.new_resources.append(page.path)
elif response.status not in [403, 404]:
self.log_red("Found webpage {0}", page.path)
self.new_resources.append(page.path)
except Timeout:
continue
except ConnectionError:
continue
except Timeout:
pass
def attack(self):
http_resources = self.persister.get_links(
attack_module=self.name) if self.do_get else []
for original_request in http_resources:
url = original_request.path
referer = original_request.referer
headers = {}
if referer:
headers["referer"] = referer
if url not in self.attacked_get:
if original_request.status in (401, 402, 403, 407):
# The ressource is forbidden
try:
evil_req = web.Request(url, method="ABC")
response = self.crawler.send(evil_req, headers=headers)
unblocked_content = response.content
if response.status == 404 or response.status < 400 or response.status >= 500:
# Every 4xx status should be uninteresting (specially bad request in our case)
self.log_red("---")
self.add_vuln(
request_id=original_request.path_id,
category=Vulnerability.HTACCESS,
level=Vulnerability.HIGH_LEVEL,
request=evil_req,
info=_(
"{0} bypassable weak restriction").format(
evil_req.url))
self.log_red(
_("Weak restriction bypass vulnerability: {0}"
), evil_req.url)
self.log_red(
_("HTTP status code changed from {0} to {1}").
format(original_request.status,
response.status))
if self.verbose == 2:
self.log_red(_("Source code:"))
self.log_red(unblocked_content)
self.log_red("---")
self.attacked_get.append(url)
except (RequestException, KeyboardInterrupt) as exception:
yield exception
yield original_request
def __init__(self, crawler, persister, logger, attack_options):
Attack.__init__(self, crawler, persister, logger, attack_options)
csv.register_dialect("nikto",
quoting=csv.QUOTE_ALL,
doublequote=False,
escapechar="\\")
user_config_dir = os.getenv("HOME") or os.getenv("USERPROFILE")
user_config_dir += "/config"
if not os.path.isdir(user_config_dir):
os.makedirs(user_config_dir)
try:
with open(os.path.join(user_config_dir, self.NIKTO_DB)) as fd:
reader = csv.reader(fd, "nikto")
self.nikto_db = [
line for line in reader
if line != [] and line[0].isdigit()
]
except IOError:
# Disable downloading of Nikto database because the license of the file
# forbids it.
self.nikto_db = []
return
try:
print(_("Problem with local nikto database."))
print(_("Downloading from the web..."))
nikto_req = web.Request(
"https://raw.githubusercontent.com/sullo/nikto/master/program/databases/db_tests"
)
response = self.crawler.send(nikto_req)
csv.register_dialect("nikto",
quoting=csv.QUOTE_ALL,
doublequote=False,
escapechar="\\")
reader = csv.reader(response.content.split("\n"), "nikto")
self.nikto_db = [
line for line in reader
if line != [] and line[0].isdigit()
]
with open(os.path.join(user_config_dir, self.NIKTO_DB),
"w") as fd:
writer = csv.writer(fd, "nikto")
writer.writerows(self.nikto_db)
except socket.timeout:
print(_("Error downloading Nikto database"))
def _get_path_by_id(self, path_id):
cursor = self._conn.cursor()
cursor.execute("SELECT * FROM paths WHERE path_id = ? LIMIT 1",
(path_id, ))
row = cursor.fetchone()
if not row:
return None
get_params = []
post_params = []
file_params = []
for param_row in cursor.execute(
"SELECT type, name, value FROM params WHERE path_id = ? ORDER BY type, param_order",
(path_id, )):
name = param_row[1]
value = param_row[2]
if param_row[0] == "GET":
get_params.append([name, value])
elif param_row[0] == "POST":
post_params.append([name, value])
else:
file_params.append([name, [value, "GIF89a", "image/gif"]])
request = web.Request(row[1],
method=row[2],
encoding=row[5],
multipart=bool(row[3]),
referer=row[8],
get_params=get_params,
post_params=post_params,
file_params=file_params)
if row[6]:
request.status = row[6]
if row[7]:
request.set_headers(json.loads(row[7]))
request.link_depth = row[4]
request.path_id = path_id
return request
def update(self):
try:
request = web.Request(self.NIKTO_DB_URL)
response = self.crawler.send(request)
csv.register_dialect("nikto",
quoting=csv.QUOTE_ALL,
doublequote=False,
escapechar="\\")
reader = csv.reader(response.content.split("\n"), "nikto")
self.nikto_db = [
line for line in reader if line != [] and line[0].isdigit()
]
with open(
os.path.join(self.persister.CRAWLER_DATA_DIR,
self.NIKTO_DB), "w") as nikto_db_file:
writer = csv.writer(nikto_db_file)
writer.writerows(self.nikto_db)
except IOError:
print(_("Error downloading nikto database."))
def __init__(self,
base_url: str,
timeout: float = 10.0,
secure: bool = False,
compression: bool = True):
self._timeout = timeout
self.stream = False
self._scope = Scope.FOLDER
self._base = web.Request(base_url)
self.auth_url = self._base.url
self.is_logged_in = False
self._user_agent = "Mozilla/5.0 (Windows NT 6.1; rv:45.0) Gecko/20100101 Firefox/45.0"
self._headers = {
"User-Agent":
self._user_agent,
"Accept-Language":
"en-US",
"Accept-Encoding":
"gzip, deflate, br",
"Accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"
}
if not compression:
self._client.headers["Accept-Encoding"] = "identity"
self._secure = secure
self._proxies = None
self._transport = None
self._drop_cookies = False
self._cookies = None
self._client = None
self._auth_credentials = {}
self._auth_method = "basic"
self._auth = None
async def async_analyze(self, request) -> Tuple[bool, List]:
async with self._sem:
self._processed_requests.append(request) # thread safe
if self._log:
print("[+] {0}".format(request))
dir_name = request.dir_name
async with self._shared_lock:
# lock to prevent launching duplicates requests that would otherwise waste time
if dir_name not in self._custom_404_codes:
invalid_page = "zqxj{0}.html".format("".join(
[choice(ascii_letters) for __ in range(10)]))
invalid_resource = web.Request(dir_name + invalid_page)
try:
page = await self._crawler.async_get(invalid_resource)
self._custom_404_codes[dir_name] = page.status
except httpx.RequestError:
pass
self._hostnames.add(request.hostname)
resource_url = request.url
try:
page = await self._crawler.async_send(request)
except (TypeError, UnicodeDecodeError) as exception:
print("{} with url {}".format(exception,
resource_url)) # debug
return False, []
# except SSLError:
# print(_("[!] SSL/TLS error occurred with URL"), resource_url)
# return False, []
# TODO: what to do of connection errors ? sleep a while before retrying ?
except ConnectionError:
print(_("[!] Connection error with URL"), resource_url)
return False, []
except httpx.RequestError as error:
print(
_("[!] {} with url {}").format(error.__class__.__name__,
resource_url))
return False, []
if self._max_files_per_dir:
async with self._shared_lock:
self._file_counts[dir_name] += 1
if self._qs_limit and request.parameters_count:
async with self._shared_lock:
self._pattern_counts[request.pattern] += 1
if request.link_depth == self._max_depth:
# We are at the edge of the depth so next links will have depth + 1 so to need to parse the page.
return True, []
# Above this line we need the content of the page. As we are in stream mode we must force reading the body.
await page.read()
# Sur les ressources statiques le content-length est généralement indiqué
if self._max_page_size > 0:
if page.raw_size > self._max_page_size:
await page.clean()
return False, []
await asyncio.sleep(0)
resources = self.extract_links(page, request)
# TODO: there's more situations where we would not want to attack the resource... must check this
if page.is_directory_redirection:
return False, resources
return True, resources
def extract_links(self, page, request) -> List:
swf_links = []
js_links = []
allowed_links = []
new_requests = []
if "application/x-shockwave-flash" in page.type or request.file_ext == "swf":
try:
swf_links = swf.extract_links_from_swf(page.bytes)
except Exception:
pass
elif "/x-javascript" in page.type or "/x-js" in page.type or "/javascript" in page.type:
js_links = lamejs.LameJs(page.content).get_links()
js_links += jsparser_angular.JsParserAngular(
page.url, page.content).get_links()
elif page.type.startswith(MIME_TEXT_TYPES):
allowed_links.extend(filter(self._crawler.is_in_scope, page.links))
allowed_links.extend(
filter(self._crawler.is_in_scope,
page.js_redirections + page.html_redirections))
for extra_url in filter(self._crawler.is_in_scope,
page.extra_urls):
parts = urlparse(extra_url)
# There are often css and js URLs with useless parameters like version or random number
# used to prevent caching in browser. So let's exclude those extensions
if parts.path.endswith(".css"):
continue
if parts.path.endswith(".js") and parts.query:
# For JS script, allow to process them but remove parameters
allowed_links.append(extra_url.split("?")[0])
continue
allowed_links.append(extra_url)
for form in page.iter_forms():
# TODO: apply bad_params filtering in form URLs
if self._crawler.is_in_scope(form):
if form.hostname not in self._hostnames:
form.link_depth = 0
else:
form.link_depth = request.link_depth + 1
new_requests.append(form)
for url in swf_links + js_links:
if url:
url = page.make_absolute(url)
if url and self._crawler.is_in_scope(url):
allowed_links.append(url)
for new_url in allowed_links:
if "?" in new_url:
path_only = new_url.split("?")[0]
if path_only not in allowed_links and self._crawler.is_in_scope(
path_only):
allowed_links.append(path_only)
for new_url in set(allowed_links):
if new_url == "":
continue
if self.is_forbidden(new_url):
continue
if "?" in new_url:
path, query_string = new_url.split("?", 1)
# TODO: encoding parameter ?
get_params = [
list(t) for t in filter(
lambda param_tuple: param_tuple[0] not in self.
_bad_params, web.parse_qsl(query_string))
]
elif new_url.endswith(EXCLUDED_MEDIA_EXTENSIONS):
# exclude static media files
continue
else:
path = new_url
get_params = []
if page.is_directory_redirection and new_url == page.redirection_url:
depth = request.link_depth
else:
depth = request.link_depth + 1
new_requests.append(
web.Request(path, get_params=get_params, link_depth=depth))
return new_requests
def attack(self):
http_resources = self.persister.get_links(
attack_module=self.name) if self.do_get else []
forms = self.persister.get_forms(
attack_module=self.name) if self.do_post else []
for original_request in chain(http_resources, forms):
# We leverage the fact that the crawler will fill password entries with a known placeholder
if "Letm3in_" not in (original_request.encoded_data +
original_request.encoded_params):
continue
# We may want to remove this but if not available fallback to target URL
if not original_request.referer:
continue
if self.verbose >= 1:
print("[+] {}".format(original_request))
request = Request(original_request.referer)
page = self.crawler.get(request, follow_redirects=True)
login_form, username_field_idx, password_field_idx = page.find_login_form(
)
if not login_form:
continue
failure_text = self.test_credentials(login_form,
username_field_idx,
password_field_idx, "invalid",
"invalid")
if self.check_success_auth(failure_text):
# Ignore this case as it raise false positives
continue
for username, password in product(self.get_usernames(),
self.get_passwords()):
response = self.test_credentials(login_form,
username_field_idx,
password_field_idx, username,
password)
if self.check_success_auth(
response) and failure_text != response:
vuln_message = _(
"Credentials found for URL {} : {} / {}").format(
original_request.referer, username, password)
# Recreate the request that succeed in order to print and store it
post_params = login_form.post_params
get_params = login_form.get_params
if login_form.method == "POST":
post_params[username_field_idx][1] = username
post_params[password_field_idx][1] = password
else:
get_params[username_field_idx][1] = username
get_params[password_field_idx][1] = password
evil_request = web.Request(
path=login_form.url,
method=login_form.method,
post_params=post_params,
get_params=get_params,
referer=login_form.referer,
link_depth=login_form.link_depth)
self.add_vuln(request_id=original_request.path_id,
category=Vulnerability.WEAK_CREDENTIALS,
level=Vulnerability.HIGH_LEVEL,
request=evil_request,
info=vuln_message)
self.log_red("---")
self.log_red(vuln_message),
self.log_red(Vulnerability.MSG_EVIL_REQUEST)
self.log_red(evil_request.http_repr())
self.log_red("---")
break
yield original_request
def explore(self, to_explore: deque, excluded_urls: list = None):
"""Explore a single TLD or the whole Web starting with an URL
@param to_explore: A list of URL to scan the scan with.
@type to_explore: list
@param excluded_urls: A list of URLs to skip. Request objects or strings which may contain wildcards.
@type excluded_urls: list
@rtype: generator
"""
invalid_page = "zqxj{0}.html".format("".join(
[choice(ascii_letters) for __ in range(10)]))
if isinstance(excluded_urls, list):
while True:
try:
bad_request = excluded_urls.pop()
except IndexError:
break
else:
if isinstance(bad_request, str):
self._regexes.append(wildcard_translate(bad_request))
elif isinstance(bad_request, web.Request):
self._excluded_requests.append(bad_request)
self._crawler._session.stream = True
if self._max_depth < 0:
raise StopIteration
while to_explore:
request = to_explore.popleft()
if not isinstance(request, web.Request):
# We treat start_urls as if they are all valid URLs (ie in scope)
request = web.Request(request, link_depth=0)
if request in self._excluded_requests:
continue
resource_url = request.url
if request.link_depth > self._max_depth:
continue
dir_name = request.dir_name
if self._max_files_per_dir and self._file_counts[
dir_name] >= self._max_files_per_dir:
continue
# Won't enter if qs_limit is 0 (aka insane mode)
if self._qs_limit:
if request.parameters_count:
try:
if self._pattern_counts[request.pattern] >= 220 / (
math.exp(request.parameters_count *
self._qs_limit)**2):
continue
except OverflowError:
# Oh boy... that's not good to try to attack a form with more than 600 input fields
# but I guess insane mode can do it as it is insane
continue
if self.is_forbidden(resource_url):
continue
if self._log:
print("[+] {0}".format(request))
if dir_name not in self._custom_404_codes:
invalid_resource = web.Request(dir_name + invalid_page)
try:
page = self._crawler.get(invalid_resource)
self._custom_404_codes[dir_name] = page.status
except RequestException:
pass
self._hostnames.add(request.hostname)
try:
page = self._crawler.send(request)
except (TypeError, UnicodeDecodeError) as exception:
print("{} with url {}".format(exception,
resource_url)) # debug
continue
except SSLError:
print(_("[!] SSL/TLS error occurred with URL"), resource_url)
continue
# TODO: what to do of connection errors ? sleep a while before retrying ?
except ConnectionError:
print(_("[!] Connection error with URL"), resource_url)
continue
except RequestException as error:
print(
_("[!] {} with url {}").format(error.__class__.__name__,
resource_url))
continue
if self._max_files_per_dir:
self._file_counts[dir_name] += 1
if self._qs_limit and request.parameters_count:
self._pattern_counts[request.pattern] += 1
self._excluded_requests.append(request)
# Sur les ressources statiques le content-length est généralement indiqué
if self._max_page_size > 0:
if page.raw_size > self._max_page_size:
page.clean()
continue
# TODO: there's more situations where we would not want to attack the resource... must check this
if not page.is_directory_redirection:
yield request
if request.link_depth == self._max_depth:
# We are at the edge of the depth so next links will have depth + 1 so to need to parse the page.
continue
accepted_urls = 0
for unfiltered_request in self.extract_links(page, request):
if BAD_URL_REGEX.search(unfiltered_request.file_path):
# Malformed link due to HTML issues
continue
if not self._crawler.is_in_scope(unfiltered_request):
continue
if unfiltered_request.hostname not in self._hostnames:
unfiltered_request.link_depth = 0
if unfiltered_request not in self._excluded_requests and unfiltered_request not in to_explore:
to_explore.append(unfiltered_request)
accepted_urls += 1
# TODO: fix this, it doesn't looks valid
# if self._max_per_depth and accepted_urls >= self._max_per_depth:
# break
self._crawler._session.stream = False
def attack(self, get_resources, forms):
"""This method searches XSS which could be permanently stored in the web application"""
for original_request in get_resources:
# First we will crawl again each webpage to look for tainted value the mod_xss module may have injected.
# So let's skip methods other than GET.
if original_request.method != "GET":
continue
url = original_request.url
target_req = web.Request(url)
referer = original_request.referer
headers = {}
if referer:
headers["referer"] = referer
if self.verbose >= 1:
print("[+] {}".format(url))
try:
response = self.crawler.send(target_req, headers=headers)
data = response.content
except Timeout:
data = ""
except OSError as exception:
data = ""
# TODO: those error messages are useless, don't give any valuable information
print(
_("error: {0} while attacking {1}").format(
exception.strerror, url))
except Exception as exception:
print(
_("error: {0} while attacking {1}").format(exception, url))
continue
# Should we look for taint codes sent with GET in the webpages?
# Exploiting those may imply sending more GET requests
if self.do_get == 1:
# Search in the page source for every taint code used by mod_xss
for taint in self.GET_XSS:
if taint in data:
# code found in the webpage !
code_url = self.GET_XSS[taint][0].url
page = self.GET_XSS[taint][0].path
parameter = self.GET_XSS[taint][1]
# Did mod_xss saw this as a reflected XSS ?
if taint in self.SUCCESSFUL_XSS:
# Yes, it means XSS payloads were injected, not just tainted code.
if self.valid_xss(data, taint,
self.SUCCESSFUL_XSS[taint]):
# If we can find the payload again, this is in fact a stored XSS
evil_request = web.Request(
code_url.replace(
taint, self.SUCCESSFUL_XSS[taint]))
self.log_red("---")
if parameter == "QUERY_STRING":
injection_msg = Vulnerability.MSG_QS_INJECT
else:
injection_msg = Vulnerability.MSG_PARAM_INJECT
self.log_red(injection_msg, self.MSG_VULN,
page, parameter)
self.log_red(Vulnerability.MSG_EVIL_URL,
code_url)
self.log_red("---")
self.add_vuln(
request_id=original_request.path_id,
category=Vulnerability.XSS,
level=Vulnerability.HIGH_LEVEL,
request=evil_request,
parameter=parameter,
info=_("Found permanent XSS in {0}"
" with {1}").format(
page, escape(evil_request.url)))
# we reported the vuln, now search another code
continue
# Ok the content is stored, but will we be able to inject javascript?
else:
timeouted = False
saw_internal_error = False
for xss, flags in self.independant_payloads:
payload = xss.replace("__XSS__", taint)
evil_request = web.Request(
code_url.replace(taint, payload))
try:
http_code = self.crawler.send(
evil_request).status
dat = self.crawler.send(target_req).content
except ReadTimeout:
dat = ""
if timeouted:
continue
self.log_orange("---")
self.log_orange(Anomaly.MSG_TIMEOUT, page)
self.log_orange(Anomaly.MSG_EVIL_REQUEST)
self.log_orange(evil_request.http_repr())
self.log_orange("---")
self.add_anom(
request_id=original_request.path_id,
category=Anomaly.RES_CONSUMPTION,
level=Anomaly.MEDIUM_LEVEL,
request=evil_request,
parameter=parameter,
info=Anomaly.MSG_PARAM_TIMEOUT.format(
parameter))
timeouted = True
except Exception as exception:
print(
_('error: {0} while attacking {1}').
format(exception, url))
continue
if self.valid_xss(dat, taint, payload):
# injection successful :)
if parameter == "QUERY_STRING":
injection_msg = Vulnerability.MSG_QS_INJECT
else:
injection_msg = Vulnerability.MSG_PARAM_INJECT
self.log_red("---")
self.log_red(injection_msg, self.MSG_VULN,
page, parameter)
self.log_red(
Vulnerability.MSG_EVIL_REQUEST)
self.log_red(evil_request.http_repr())
self.log_red("---")
self.add_vuln(
request_id=original_request.path_id,
category=Vulnerability.XSS,
level=Vulnerability.HIGH_LEVEL,
request=evil_request,
parameter=parameter,
info=_("Found permanent XSS in {0}"
" with {1}").format(
url,
escape(evil_request.url)))
# look for another code in the webpage
break
elif http_code == 500 and not saw_internal_error:
self.add_anom(
request_id=original_request.path_id,
category=Anomaly.ERROR_500,
level=Anomaly.HIGH_LEVEL,
request=evil_request,
parameter=parameter,
info=Anomaly.MSG_PARAM_500.format(
parameter))
self.log_orange("---")
self.log_orange(Anomaly.MSG_500, page)
self.log_orange(Anomaly.MSG_EVIL_REQUEST)
self.log_orange(evil_request.http_repr())
self.log_orange("---")
saw_internal_error = True
# Should we look for taint codes sent with POST in the webpages?
# Exploiting those may probably imply sending more POST requests
if self.do_post == 1:
for taint in self.POST_XSS:
if taint in data:
# Code found in the webpage!
# Did mod_xss saw this as a reflected XSS ?
if taint in self.SUCCESSFUL_XSS:
if self.valid_xss(data, taint,
self.SUCCESSFUL_XSS[taint]):
code_req = self.POST_XSS[taint][0]
get_params = code_req.get_params
post_params = code_req.post_params
file_params = code_req.file_params
referer = code_req.referer
for params_list in [
get_params, post_params, file_params
]:
for i in range(len(params_list)):
parameter, value = params_list[i]
parameter = quote(parameter)
if value == taint:
if params_list is file_params:
params_list[i][1][
0] = self.SUCCESSFUL_XSS[
taint]
else:
params_list[i][
1] = self.SUCCESSFUL_XSS[
taint]
# we found the xss payload again -> stored xss vuln
evil_request = web.Request(
code_req.path,
method="POST",
get_params=get_params,
post_params=post_params,
file_params=file_params,
referer=referer)
self.add_vuln(
request_id=original_request.
path_id,
category=Vulnerability.XSS,
level=Vulnerability.HIGH_LEVEL,
request=evil_request,
parameter=parameter,
info=_(
"Found permanent XSS attacked by {0} with fields"
" {1}").format(
evil_request.url,
encode(post_params)))
self.log_red("---")
self.log_red(
Vulnerability.MSG_PARAM_INJECT,
self.MSG_VULN,
evil_request.path, parameter)
self.log_red(
Vulnerability.MSG_EVIL_REQUEST)
self.log_red(
evil_request.http_repr())
self.log_red("---")
# search for the next code in the webpage
continue
# we found the code but no attack was made
# let's try to break in
else:
code_req = self.POST_XSS[taint][0]
get_params = code_req.get_params
post_params = code_req.post_params
file_params = code_req.file_params
referer = code_req.referer
for params_list in [
get_params, post_params, file_params
]:
for i in range(len(params_list)):
parameter, value = params_list[i]
parameter = quote(parameter)
if value == taint:
timeouted = False
saw_internal_error = False
for xss, flags in self.independant_payloads:
payload = xss.replace(
"__XSS__", taint)
if params_list is file_params:
params_list[i][1][0] = payload
else:
params_list[i][1] = payload
try:
evil_request = web.Request(
code_req.path,
method=code_req.method,
get_params=get_params,
post_params=post_params,
file_params=file_params,
referer=referer)
http_code = self.crawler.send(
evil_request).status
dat = self.crawler.send(
target_req).content
except ReadTimeout:
dat = ""
if timeouted:
continue
self.log_orange("---")
self.log_orange(
Anomaly.MSG_TIMEOUT,
evil_request.url)
self.log_orange(
Anomaly.MSG_EVIL_REQUEST)
self.log_orange(
evil_request.http_repr())
self.log_orange("---")
self.add_anom(
request_id=original_request
.path_id,
category=Anomaly.
RES_CONSUMPTION,
level=Anomaly.MEDIUM_LEVEL,
request=evil_request,
parameter=parameter,
info=Anomaly.
MSG_PARAM_TIMEOUT.format(
parameter))
timeouted = True
except Exception as exception:
print(
_("error: {0} while attacking {1}"
).format(exception, url))
continue
if self.valid_xss(
dat, taint, payload):
self.add_vuln(
request_id=original_request
.path_id,
category=Vulnerability.XSS,
level=Vulnerability.
HIGH_LEVEL,
request=evil_request,
parameter=parameter,
info=
_("Found permanent XSS attacked by {0} with fields"
" {1}").format(
evil_request.url,
encode(post_params)))
self.log_red("---")
self.log_red(
Vulnerability.
MSG_PARAM_INJECT,
self.MSG_VULN,
evil_request.path,
parameter)
self.log_red(Vulnerability.
MSG_EVIL_REQUEST)
self.log_red(
evil_request.http_repr())
self.log_red("---")
break
elif http_code == 500 and not saw_internal_error:
self.add_anom(
request_id=original_request
.path_id,
category=Anomaly.ERROR_500,
level=Anomaly.HIGH_LEVEL,
request=evil_request,
parameter=parameter,
info=Anomaly.MSG_PARAM_500.
format(parameter))
self.log_orange("---")
self.log_orange(
Anomaly.MSG_500,
evil_request.url)
self.log_orange(
Anomaly.MSG_EVIL_REQUEST)
self.log_orange(
evil_request.http_repr())
self.log_orange("---")
saw_internal_error = True
yield original_request
def attack(self):
"""This method searches XSS which could be permanently stored in the web application"""
get_resources = self.persister.get_links(attack_module=self.name) if self.do_get else []
for original_request in get_resources:
if not valid_xss_content_type(original_request) or original_request.status in (301, 302, 303):
# If that content-type can't be interpreted as HTML by browsers then it is useless
# Same goes for redirections
continue
url = original_request.url
target_req = web.Request(url)
referer = original_request.referer
headers = {}
if referer:
headers["referer"] = referer
if self.verbose >= 1:
print("[+] {}".format(url))
try:
response = self.crawler.send(target_req, headers=headers)
data = response.content
except Timeout:
continue
except OSError as exception:
# TODO: those error messages are useless, don't give any valuable information
print(_("error: {0} while attacking {1}").format(exception.strerror, url))
continue
except Exception as exception:
print(_("error: {0} while attacking {1}").format(exception, url))
continue
# Should we look for taint codes sent with GET in the webpages?
# Exploiting those may imply sending more GET requests
# Search in the page source for every taint code used by mod_xss
for taint in self.TRIED_XSS:
input_request = self.TRIED_XSS[taint][0]
# Such situations should not occur as it would be stupid to block POST (or GET) requests for mod_xss
# and not mod_permanentxss, but it is possible so let's filter that.
if not self.do_get and input_request.method == "GET":
continue
if not self.do_post and input_request.method == "POST":
continue
if taint.lower() in data.lower():
# Code found in the webpage !
# Did mod_xss saw this as a reflected XSS ?
if taint in self.SUCCESSFUL_XSS:
# Yes, it means XSS payloads were injected, not just tainted code.
payload, flags = self.SUCCESSFUL_XSS[taint]
if self.check_payload(response, flags, taint):
# If we can find the payload again, this is in fact a stored XSS
get_params = input_request.get_params
post_params = input_request.post_params
file_params = input_request.file_params
referer = input_request.referer
# The following trick may seems dirty but it allows to treat GET and POST requests
# the same way.
for params_list in [get_params, post_params, file_params]:
for i in range(len(params_list)):
parameter, value = params_list[i]
parameter = quote(parameter)
if value != taint:
continue
if params_list is file_params:
params_list[i][1][0] = payload
else:
params_list[i][1] = payload
# we found the xss payload again -> stored xss vuln
evil_request = web.Request(
input_request.path,
method=input_request.method,
get_params=get_params,
post_params=post_params,
file_params=file_params,
referer=referer
)
if original_request.path == input_request.path:
description = _(
"Permanent XSS vulnerability found via injection in the parameter {0}"
).format(parameter)
else:
description = _(
"Permanent XSS vulnerability found in {0} by injecting"
" the parameter {1} of {2}"
).format(
original_request.url,
parameter,
input_request.path
)
if has_csp(response):
description += ".\n" + _("Warning: Content-Security-Policy is present!")
self.add_vuln(
request_id=original_request.path_id,
category=Vulnerability.XSS,
level=Vulnerability.HIGH_LEVEL,
request=evil_request,
parameter=parameter,
info=description
)
if parameter == "QUERY_STRING":
injection_msg = Vulnerability.MSG_QS_INJECT
else:
injection_msg = Vulnerability.MSG_PARAM_INJECT
self.log_red("---")
self.log_red(
injection_msg,
self.MSG_VULN,
original_request.path,
parameter
)
if has_csp(response):
self.log_red(_("Warning: Content-Security-Policy is present!"))
self.log_red(Vulnerability.MSG_EVIL_REQUEST)
self.log_red(evil_request.http_repr())
self.log_red("---")
# FIX: search for the next code in the webpage
# Ok the content is stored, but will we be able to inject javascript?
else:
parameter = self.TRIED_XSS[taint][1]
payloads = generate_payloads(response.content, taint, self.independant_payloads)
flags = self.TRIED_XSS[taint][2]
# TODO: check that and make it better
if PayloadType.get in flags:
method = "G"
elif PayloadType.file in flags:
method = "F"
else:
method = "P"
self.attempt_exploit(method, payloads, input_request, parameter, taint, original_request)
yield original_request
async def _get_paths(self,
path=None,
method=None,
crawled: bool = True,
module: str = "",
evil: bool = False):
conditions = [paths.c.evil == evil]
if path and isinstance(path, str):
conditions.append(paths.c.path == path)
if method in ("GET", "POST"):
conditions.append(paths.c.method == method)
if crawled:
conditions.append(paths.c.headers != None)
async with self._engine.begin() as conn:
result = await conn.execute(
select(paths).where(and_(True,
*conditions)).order_by(paths.c.path))
for row in result.fetchall():
path_id = row[0]
if module:
# Exclude requests matching the attack module, we want requests that aren't attacked yet
statement = select(attack_logs).where(
attack_logs.c.path_id == path_id,
attack_logs.c.module == module).limit(1)
async with self._engine.begin() as conn:
result = await conn.execute(statement)
if result.fetchone():
continue
get_params = []
post_params = []
file_params = []
statement = select(
params.c.type, params.c.name, params.c.value1, params.c.value2,
params.c.meta).where(params.c.path_id == path_id).order_by(
params.c.type, params.c.position)
async with self._engine.begin() as conn:
async_result = await conn.stream(statement)
async for param_row in async_result:
name = param_row[1]
value1 = param_row[2]
if param_row[0] == "GET":
get_params.append([name, value1])
elif param_row[0] == "POST":
if name == "__RAW__" and not post_params:
# First POST parameter is __RAW__, it should mean that we have raw content
post_params = value1
elif isinstance(post_params, list):
post_params.append([name, value1])
elif param_row[0] == "FILE":
if param_row[4]:
file_params.append(
[name, (value1, param_row[3], param_row[4])])
else:
file_params.append([name, (value1, param_row[3])])
else:
raise ValueError("Unknown param type {}".format(
param_row[0]))
http_res = web.Request(row[1],
method=row[2],
encoding=row[5],
enctype=row[3],
referer=row[8],
get_params=get_params,
post_params=post_params,
file_params=file_params)
if row[6]:
http_res.status = row[6]
if row[7]:
http_res.set_headers(row[7])
http_res.link_depth = row[4]
http_res.path_id = path_id
yield http_res
def _get_paths(self,
path=None,
method=None,
crawled: bool = True,
attack_module: str = "",
evil: bool = False):
cursor = self._conn.cursor()
conditions = ["evil = ?"]
args = [int(evil)]
if path and isinstance(path, str):
conditions.append("path = ?")
args.append(path)
if method in ("GET", "POST"):
conditions.append("method = ?")
args.append(method)
if crawled:
conditions.append("headers IS NOT NULL")
conditions = " AND ".join(conditions)
conditions = "WHERE " + conditions
cursor.execute(
"SELECT * FROM paths {} ORDER BY path".format(conditions), args)
for row in cursor.fetchall():
path_id = row[0]
if attack_module:
# Exclude requests matching the attack module, we want requests that aren't attacked yet
cursor.execute(
"SELECT * FROM attack_log WHERE path_id = ? AND module_name = ? LIMIT 1",
(path_id, attack_module))
if cursor.fetchone():
continue
get_params = []
post_params = []
file_params = []
for param_row in cursor.execute(
("SELECT type, name, value1, value2, meta "
"FROM params "
"WHERE path_id = ? "
"ORDER BY type, param_order"), (path_id, )):
name = param_row[1]
value1 = param_row[2]
if param_row[0] == "GET":
get_params.append([name, value1])
elif param_row[0] == "POST":
if name == "__RAW__" and not post_params:
# First POST parameter is __RAW__, it should mean that we have raw content
post_params = value1
elif isinstance(post_params, list):
post_params.append([name, value1])
elif param_row[0] == "FILE":
if param_row[4]:
file_params.append(
[name, (value1, param_row[3], param_row[4])])
else:
file_params.append([name, (value1, param_row[3])])
else:
raise ValueError("Unknown param type {}".format(
param_row[0]))
http_res = web.Request(row[1],
method=row[2],
encoding=row[5],
enctype=row[3],
referer=row[8],
get_params=get_params,
post_params=post_params,
file_params=file_params)
if row[6]:
http_res.status = row[6]
if row[7]:
http_res.set_headers(json.loads(row[7]))
http_res.link_depth = row[4]
http_res.path_id = path_id
yield http_res
async def async_explore(self,
to_explore: deque,
excluded_urls: list = None):
"""Explore a single TLD or the whole Web starting with an URL
@param to_explore: A list of URL to scan the scan with.
@type to_explore: list
@param excluded_urls: A list of URLs to skip. Request objects or strings which may contain wildcards.
@type excluded_urls: list
@rtype: generator
"""
if isinstance(excluded_urls, list):
while True:
try:
bad_request = excluded_urls.pop()
except IndexError:
break
else:
if isinstance(bad_request, str):
self._regexes.append(wildcard_translate(bad_request))
elif isinstance(bad_request, web.Request):
self._processed_requests.append(bad_request)
self._crawler.stream = True
if self._max_depth < 0:
raise StopIteration
task_to_request = {}
while True:
while to_explore:
# Concurrent tasks are limited through the use of the semaphore BUT we don't want the to_explore
# queue to be empty everytime (as we may need to extract remaining URLs) and overload the event loop
# with pending tasks.
# There may be more suitable way to do this though.
if len(task_to_request) > self._max_tasks:
break
if self._stopped.is_set():
break
request = to_explore.popleft()
if not isinstance(request, web.Request):
# We treat start_urls as if they are all valid URLs (ie in scope)
request = web.Request(request, link_depth=0)
if request in self._processed_requests:
continue
resource_url = request.url
if request.link_depth > self._max_depth:
continue
dir_name = request.dir_name
if self._max_files_per_dir and self._file_counts[
dir_name] >= self._max_files_per_dir:
continue
# Won't enter if qs_limit is 0 (aka insane mode)
if self._qs_limit:
if request.parameters_count:
try:
if self._pattern_counts[request.pattern] >= 220 / (
math.exp(request.parameters_count *
self._qs_limit)**2):
continue
except OverflowError:
# Oh boy... that's not good to try to attack a form with more than 600 input fields
# but I guess insane mode can do it as it is insane
continue
if self.is_forbidden(resource_url):
continue
task = asyncio.create_task(self.async_analyze(request))
task_to_request[task] = request
if task_to_request:
done, __ = await asyncio.wait(
task_to_request,
timeout=0.25,
return_when=asyncio.FIRST_COMPLETED)
else:
done = []
# process any completed task
for task in done:
request = task_to_request[task]
try:
success, resources = await task
except Exception as exc:
print('%r generated an exception: %s' % (request, exc))
else:
if success:
yield request
accepted_urls = 0
for unfiltered_request in resources:
if BAD_URL_REGEX.search(unfiltered_request.file_path):
# Malformed link due to HTML issues
continue
if not self._crawler.is_in_scope(unfiltered_request):
continue
if unfiltered_request.hostname not in self._hostnames:
unfiltered_request.link_depth = 0
if unfiltered_request not in self._processed_requests and unfiltered_request not in to_explore:
to_explore.append(unfiltered_request)
accepted_urls += 1
# TODO: fix this, it doesn't looks valid
# if self._max_per_depth and accepted_urls >= self._max_per_depth:
# break
# remove the now completed task
del task_to_request[task]
if not task_to_request and (self._stopped.is_set()
or not to_explore):
break
self._crawler.stream = False
def iter_forms(self, autofill=True):
"""Returns a generator of Request extracted from the Page.
@rtype: generator
"""
for form in self.soup.find_all("form"):
url = self.make_absolute(form.attrs.get("action", "").strip() or self.url)
# If no method is specified then it's GET. If an invalid method is set it's GET.
method = "POST" if form.attrs.get("method", "GET").upper() == "POST" else "GET"
enctype = "" if method == "GET" else form.attrs.get("enctype", "application/x-www-form-urlencoded").lower()
get_params = []
post_params = []
# If the form must be sent in multipart, everything should be given to requests in the files parameter
# but internally we use the file_params list only for file inputs sent with multipart (as they must be
# threated differently in persister). Crawler.post() method will join post_params and file_params for us
# if the enctype is multipart.
file_params = []
form_actions = set()
defaults = {
"checkbox": "default",
"color": "#bada55",
"date": "2019-03-03",
"datetime": "2019-03-03T20:35:34.32",
"datetime-local": "2019-03-03T22:41",
"email": "*****@*****.**",
"file": ("pix.gif", "GIF89a", "image/gif"),
"hidden": "default",
"month": "2019-03",
"number": "1337",
"password": "******", # 8 characters with uppercase, digit and special char for common rules
"radio": "beton", # priv8 j0k3
"range": "37",
"search": "default",
"submit": "submit",
"tel": "0606060606",
"text": "default",
"time": "13:37",
"url": "https://wapiti.sourceforge.io/",
"username": "******",
"week": "2019-W24"
}
radio_inputs = {}
for input_field in form.find_all("input", attrs={"name": True}):
input_type = input_field.attrs.get("type", "text").lower()
if input_type in {"reset", "button"}:
# Those input types doesn't send any value
continue
if input_type == "image":
if method == "GET":
get_params.append([input_field["name"] + ".x", "1"])
get_params.append([input_field["name"] + ".y", "1"])
else:
post_params.append([input_field["name"] + ".x", "1"])
post_params.append([input_field["name"] + ".y", "1"])
elif input_type in defaults:
if input_type == "text" and "mail" in input_field["name"] and autofill:
# If an input text match name "mail" then put a valid email address in it
input_value = defaults["email"]
elif input_type == "text" and "pass" in input_field["name"] or \
"pwd" in input_field["name"] and autofill:
# Looks like a text field but waiting for a password
input_value = defaults["password"]
elif input_type == "text" and "user" in input_field["name"] or \
"login" in input_field["name"] and autofill:
input_value = defaults["username"]
else:
input_value = input_field.get("value", defaults[input_type] if autofill else "")
if input_type == "file":
# With file inputs the content is only sent if the method is POST and enctype multipart
# otherwise only the file name is sent.
# Having a default value set in HTML for a file input doesn't make sense... force our own.
if method == "GET":
get_params.append([input_field["name"], "pix.gif"])
else:
if "multipart" in enctype:
file_params.append([input_field["name"], defaults["file"]])
else:
post_params.append([input_field["name"], "pix.gif"])
elif input_type == "radio":
# Do not put in forms now, do it at the end
radio_inputs[input_field["name"]] = input_value
else:
if method == "GET":
get_params.append([input_field["name"], input_value])
else:
post_params.append([input_field["name"], input_value])
# A formaction doesn't need a name
for input_field in form.find_all("input", attrs={"formaction": True}):
form_actions.add(self.make_absolute(input_field["formaction"].strip() or self.url))
for button_field in form.find_all("button", formaction=True):
# If formaction is empty it basically send to the current URL
# which can be different from the defined action attribute on the form...
form_actions.add(self.make_absolute(button_field["formaction"].strip() or self.url))
if form.find("input", attrs={"name": False, "type": "image"}):
# Unnamed input type file => names will be set as x and y
if method == "GET":
get_params.append(["x", "1"])
get_params.append(["y", "1"])
else:
post_params.append(["x", "1"])
post_params.append(["y", "1"])
for select in form.find_all("select", attrs={"name": True}):
all_values = []
selected_value = None
for option in select.find_all("option", value=True):
all_values.append(option["value"])
if "selected" in option.attrs:
selected_value = option["value"]
if selected_value is None and all_values:
# First value may be a placeholder but last entry should be valid
selected_value = all_values[-1]
if method == "GET":
get_params.append([select["name"], selected_value])
else:
post_params.append([select["name"], selected_value])
# if form.find("input", attrs={"type": "image", "name": False}):
# new_form.add_image_field()
for text_area in form.find_all("textarea", attrs={"name": True}):
if method == "GET":
get_params.append([text_area["name"], "Hi there!" if autofill else ""])
else:
post_params.append([text_area["name"], "Hi there!" if autofill else ""])
# I guess I should raise a new form for every possible radio values...
# For the moment, just use the last value
for radio_name, radio_value in radio_inputs.items():
if method == "GET":
get_params.append([radio_name, radio_value])
else:
post_params.append([radio_name, radio_value])
if method == "POST" and not post_params and not file_params:
# Ignore empty forms. Those are either webdev issues or forms having only "button" types that
# only rely on JS code.
continue
# First raise the form with the URL specified in the action attribute
new_form = web.Request(
url,
method=method,
get_params=get_params,
post_params=post_params,
file_params=file_params,
encoding=self.apparent_encoding,
referer=self.url,
enctype=enctype
)
yield new_form
# Then if we saw some formaction attribute, raise the form with the given formaction URL
for url in form_actions:
new_form = web.Request(
url,
method=method,
get_params=get_params,
post_params=post_params,
file_params=file_params,
encoding=self.apparent_encoding,
referer=self.url,
enctype=enctype
)
yield new_form
def _get_paths(self,
path=None,
method=None,
crawled: bool = True,
attack_module: str = "",
evil: bool = False):
cursor = self._conn.cursor()
conditions = ["evil = ?"]
args = [int(evil)]
if path and isinstance(path, str):
conditions.append("path = ?")
args.append(path)
if method in ("GET", "POST"):
conditions.append("method = ?")
args.append(method)
if crawled:
conditions.append("headers IS NOT NULL")
conditions = " AND ".join(conditions)
conditions = "WHERE " + conditions
cursor.execute(
"SELECT * FROM paths {} ORDER BY path".format(conditions), args)
for row in cursor.fetchall():
path_id = row[0]
if attack_module:
# Exclude requests matching the attack module, we wan't requests that aren't attacked yet
cursor.execute(
"SELECT * FROM attack_log WHERE path_id = ? AND module_name = ? LIMIT 1",
(path_id, attack_module))
if cursor.fetchone():
continue
get_params = []
post_params = []
file_params = []
for param_row in cursor.execute(
"SELECT type, name, value FROM params WHERE path_id = ? ORDER BY type, param_order",
(path_id, )):
name = param_row[1]
value = param_row[2]
if param_row[0] == "GET":
get_params.append([name, value])
elif param_row[0] == "POST":
post_params.append([name, value])
else:
file_params.append([name, [value, "GIF89a", "image/gif"]])
http_res = web.Request(row[1],
method=row[2],
encoding=row[5],
multipart=bool(row[3]),
referer=row[8],
get_params=get_params,
post_params=post_params,
file_params=file_params)
if row[6]:
http_res.status = row[6]
if row[7]:
http_res.set_headers(json.loads(row[7]))
http_res.link_depth = row[4]
http_res.path_id = path_id
yield http_res
async def get_path_by_id(self, path_id):
path_id = int(path_id)
async with self._engine.begin() as conn:
result = await conn.execute(
select(paths).where(paths.c.path_id == path_id).limit(1))
row = result.fetchone()
if not row:
return None
get_params = []
post_params = []
file_params = []
statement = select(
params.c.type, params.c.name, params.c.value1, params.c.value2,
params.c.meta).where(params.c.path_id == path_id).order_by(
params.c.type, params.c.position)
async with self._engine.begin() as conn:
async_result = await conn.stream(statement)
async for param_row in async_result:
name = param_row[1]
value1 = param_row[2]
if param_row[0] == "GET":
get_params.append([name, value1])
elif param_row[0] == "POST":
if name == "__RAW__" and not post_params:
# First POST parameter is __RAW__, it should mean that we have raw content
post_params = value1
elif isinstance(post_params, list):
post_params.append([name, value1])
elif param_row[0] == "FILE":
if param_row[4]:
file_params.append(
[name, (value1, param_row[3], param_row[4])])
else:
file_params.append([name, (value1, param_row[3])])
else:
raise ValueError("Unknown param type {}".format(
param_row[0]))
request = web.Request(row[1],
method=row[2],
encoding=row[5],
enctype=row[3],
referer=row[8],
get_params=get_params,
post_params=post_params,
file_params=file_params)
if row[6]:
request.status = row[6]
if row[7]:
request.set_headers(row[7])
request.link_depth = row[4]
request.path_id = path_id
return request
def attack(self):
junk_string = "w" + "".join([
random.choice("0123456789abcdefghjijklmnopqrstuvwxyz")
for __ in range(0, 5000)
])
urls = self.persister.get_links(
attack_module=self.name) if self.do_get else []
server = next(urls).hostname
for line in self.nikto_db:
match = match_or = match_and = False
fail = fail_or = False
osv_id = line[1]
path = line[3]
method = line[4]
vuln_desc = line[10]
post_data = line[11]
path = path.replace("@CGIDIRS", "/cgi-bin/")
path = path.replace("@ADMIN", "/admin/")
path = path.replace("@NUKE", "/modules/")
path = path.replace("@PHPMYADMIN", "/phpMyAdmin/")
path = path.replace("@POSTNUKE", "/postnuke/")
path = re.sub(r"JUNK\((\d+)\)",
lambda x: junk_string[:int(x.group(1))], path)
if path[0] == "@":
continue
if not path.startswith("/"):
path = "/" + path
try:
url = "http://" + server + path
except UnicodeDecodeError:
continue
if method == "GET":
evil_request = web.Request(url)
elif method == "POST":
evil_request = web.Request(url,
post_params=post_data,
method=method)
else:
evil_request = web.Request(url,
post_params=post_data,
method=method)
if self.verbose == 2:
if method == "GET":
print("[¨] {0}".format(evil_request.url))
else:
print("[¨] {0}".format(evil_request.http_repr()))
try:
response = self.crawler.send(evil_request)
except RequestException as exception:
# requests bug
yield exception
continue
except ValueError:
# ValueError raised by urllib3 (Method cannot contain non-token characters), we don't want to raise
yield
else:
yield
page = response.content
code = response.status
raw = " ".join([x + ": " + y for x, y in response.headers.items()])
raw += page
# First condition (match)
if len(line[5]) == 3 and line[5].isdigit():
if code == int(line[5]):
match = True
else:
if line[5] in raw:
match = True
# Second condition (or)
if line[6] != "":
if len(line[6]) == 3 and line[6].isdigit():
if code == int(line[6]):
match_or = True
else:
if line[6] in raw:
match_or = True
# Third condition (and)
if line[7] != "":
if len(line[7]) == 3 and line[7].isdigit():
if code == int(line[7]):
match_and = True
else:
if line[7] in raw:
match_and = True
else:
match_and = True
# Fourth condition (fail)
if line[8] != "":
if len(line[8]) == 3 and line[8].isdigit():
if code == int(line[8]):
fail = True
else:
if line[8] in raw:
fail = True
# Fifth condition (or)
if line[9] != "":
if len(line[9]) == 3 and line[9].isdigit():
if code == int(line[9]):
fail_or = True
else:
if line[9] in raw:
fail_or = True
if ((match or match_or) and match_and) and not (fail or fail_or):
self.log_red("---")
self.log_red(vuln_desc)
self.log_red(url)
refs = []
if osv_id != "0":
refs.append("http://osvdb.org/show/osvdb/" + osv_id)
# CERT
cert_advisory = re.search("(CA-[0-9]{4}-[0-9]{2})", vuln_desc)
if cert_advisory is not None:
refs.append("http://www.cert.org/advisories/" +
cert_advisory.group(0) + ".html")
# SecurityFocus
securityfocus_bid = re.search("BID-([0-9]{4})", vuln_desc)
if securityfocus_bid is not None:
refs.append("http://www.securityfocus.com/bid/" +
securityfocus_bid.group(1))
# Mitre.org
mitre_cve = re.search("((CVE|CAN)-[0-9]{4}-[0-9]{4,})",
vuln_desc)
if mitre_cve is not None:
refs.append(
"http://cve.mitre.org/cgi-bin/cvename.cgi?name=" +
mitre_cve.group(0))
# CERT Incidents
cert_incident = re.search("(IN-[0-9]{4}-[0-9]{2})", vuln_desc)
if cert_incident is not None:
refs.append("http://www.cert.org/incident_notes/" +
cert_incident.group(0) + ".html")
# Microsoft Technet
ms_bulletin = re.search("(MS[0-9]{2}-[0-9]{3})", vuln_desc)
if ms_bulletin is not None:
refs.append(
"http://www.microsoft.com/technet/security/bulletin/" +
ms_bulletin.group(0) + ".asp")
info = vuln_desc
if refs:
self.log_red(_("References:"))
self.log_red(" {0}".format("\n ".join(refs)))
info += "\n" + _("References:") + "\n"
info += "\n".join(refs)
self.log_red("---")
self.add_vuln(category=Vulnerability.NIKTO,
level=Vulnerability.HIGH_LEVEL,
request=evil_request,
info=info)