def url_matches(self, request_uri):
"""
:param request_uri: The http request URI sent by the plugin
:return: True if the request_uri matches this mock_response
"""
if isinstance(self.url, basestring):
request_uri = URL(request_uri)
response_uri = URL(self.url)
request_path = request_uri.get_path_qs()
request_domain = request_uri.get_domain()
response_path = response_uri.get_path_qs()
response_domain = response_uri.get_domain()
if response_domain != request_domain:
return False
if request_path != response_path:
return False
return True
elif isinstance(self.url, RE_COMPILE_TYPE):
if self.url.match(request_uri):
return True
return False
def url_matches(self, request_uri):
"""
:param request_uri: The http request URI sent by the plugin
:return: True if the request_uri matches this mock_response
"""
if isinstance(self.url, basestring):
request_uri = URL(request_uri)
response_uri = URL(self.url)
request_path = request_uri.get_path_qs()
request_domain = request_uri.get_domain()
response_path = response_uri.get_path_qs()
response_domain = response_uri.get_domain()
if response_domain != request_domain:
return False
if request_path != response_path:
return False
return True
elif isinstance(self.url, RE_COMPILE_TYPE):
if self.url.match(request_uri):
return True
return False
def test_set_domain(self):
u = URL('http://w3af.com/def/jkl/')
self.assertEqual(u.get_domain(), 'w3af.com')
u.set_domain('host.tld')
self.assertEqual(u.get_domain(), 'host.tld')
u.set_domain('foobar')
self.assertEqual(u.get_domain(), 'foobar')
u.set_domain('foobar.')
self.assertEqual(u.get_domain(), 'foobar.')
def test_set_domain(self):
u = URL('http://w3af.com/def/jkl/')
self.assertEqual(u.get_domain(), 'w3af.com')
u.set_domain('host.tld')
self.assertEqual(u.get_domain(), 'host.tld')
u.set_domain('foobar')
self.assertEqual(u.get_domain(), 'foobar')
u.set_domain('foobar.')
self.assertEqual(u.get_domain(), 'foobar.')
def test_websocket_secure_proto(self):
"""
We can also parse and handle ws and wss protocols
"""
u = URL('wss://w3af.com')
self.assertEqual(u.get_domain(), 'w3af.com')
self.assertEqual(u.get_protocol(), 'wss')
def test_default_proto(self):
"""
http is the default protocol, we can provide URLs with no proto
"""
u = URL('w3af.com')
self.assertEqual(u.get_domain(), 'w3af.com')
self.assertEqual(u.get_protocol(), 'http')
def setUp(self):
self.kb.cleanup()
self.w3afcore = w3afCore()
self.misc_settings = MiscSettings()
self.request_callback_call_count = 0
self.request_callback_match = 0
if self.MOCK_RESPONSES:
httpretty.reset()
httpretty.enable()
try:
url = URL(self.target_url)
except ValueError, ve:
msg = ('When using MOCK_RESPONSES you need to set the'
' target_url attribute to a valid URL, exception was:'
' "%s".')
raise Exception(msg % ve)
domain = url.get_domain()
proto = url.get_protocol()
port = url.get_port()
self._register_httpretty_uri(proto, domain, port)
def test_websocket_secure_proto(self):
"""
We can also parse and handle ws and wss protocols
"""
u = URL('wss://w3af.com')
self.assertEqual(u.get_domain(), 'w3af.com')
self.assertEqual(u.get_protocol(), 'wss')
def test_default_proto(self):
"""
http is the default protocol, we can provide URLs with no proto
"""
u = URL('w3af.com')
self.assertEqual(u.get_domain(), 'w3af.com')
self.assertEqual(u.get_protocol(), 'http')
def __init__(self,
url,
body,
content_type='text/html',
status=200,
method='GET',
headers=None,
delay=None):
self.url = url
self.body = body
self.status = status
self.method = method
self.delay = delay
self.content_type = content_type
self.headers = {'Content-Type': content_type}
if headers is not None:
self.headers.update(headers)
assert method in self.KNOWN_METHODS, self.NO_MOCK
assert isinstance(url, (basestring, RE_COMPILE_TYPE))
if isinstance(url, basestring):
url = URL(url)
assert url.get_domain(), 'Need to specify the MockResponse domain'
def search(self, query, start, count=10):
"""
Search the web with Bing.
This method is based from the msn.py file from the massive enumeration
toolset, coded by pdp and released under GPL v2.
"""
url = 'http://www.bing.com/search?'
query = urllib.urlencode({
'q': query,
'first': start + 1,
'FORM': 'PERE'
})
url_instance = URL(url + query)
response = self._uri_opener.GET(url_instance,
headers=self._headers,
cache=True,
grep=False,
follow_redirects=True)
# This regex might become outdated, but the good thing is that we have
# test_bing.py which is going to fail and tell us that it's outdated
re_match = re.findall('<a href="((http|https)(.*?))" h="ID=SERP,',
response.get_body())
results = set()
for url, _, _ in re_match:
try:
url = URL(url)
except ValueError:
pass
else:
# Test for full match.
if url.get_domain() not in self.BLACKLISTED_DOMAINS:
# Now test for partial match
for blacklisted_domain in self.BLACKLISTED_DOMAINS:
if blacklisted_domain in url.get_domain():
# ignore this domain.
break
else:
bing_result = BingResult(url)
results.add(bing_result)
return results
def _url_matches(self, phishing_url, phishtank_detail_url):
"""
:param url: The url (as string) from the phishtank database
:return: A PhishTankMatch if url matches what we're looking for, None
if there is no match
"""
for query_result in self._multi_in.query(phishing_url):
phish_url = URL(phishing_url)
target_host_url = URL(query_result[0])
if target_host_url.get_domain() == phish_url.get_domain() or \
phish_url.get_domain().endswith('.' + target_host_url.get_domain()):
phish_detail_url = URL(phishtank_detail_url)
ptm = PhishTankMatch(phish_url, phish_detail_url)
return ptm
return None
def _url_matches(self, phishing_url, phishtank_detail_url):
"""
:param url: The url (as string) from the phishtank database
:return: A PhishTankMatch if url matches what we're looking for, None
if there is no match
"""
for query_result in self._multi_in.query(phishing_url):
phish_url = URL(phishing_url)
target_host_url = URL(query_result[0])
if target_host_url.get_domain() == phish_url.get_domain() or \
phish_url.get_domain().endswith('.' + target_host_url.get_domain()):
phish_detail_url = URL(phishtank_detail_url)
ptm = PhishTankMatch(phish_url, phish_detail_url)
return ptm
return None
def search(self, query, start, count=10):
"""
Search the web with Bing.
This method is based from the msn.py file from the massive enumeration
toolset, coded by pdp and released under GPL v2.
"""
url = 'http://www.bing.com/search?'
query = urllib.urlencode({'q': query,
'first': start + 1,
'FORM': 'PERE'})
url_instance = URL(url + query)
response = self._uri_opener.GET(url_instance, headers=self._headers,
cache=True, grep=False,
follow_redirects=True)
# This regex might become outdated, but the good thing is that we have
# test_bing.py which is going to fail and tell us that it's outdated
re_match = re.findall('<a href="((http|https)(.*?))" h="ID=SERP,',
response.get_body())
results = set()
for url, _, _ in re_match:
try:
url = URL(url)
except ValueError:
pass
else:
# Test for full match.
if url.get_domain() not in self.BLACKLISTED_DOMAINS:
# Now test for partial match
for blacklisted_domain in self.BLACKLISTED_DOMAINS:
if blacklisted_domain in url.get_domain():
# ignore this domain.
break
else:
bing_result = BingResult(url)
results.add(bing_result)
return results
def test_phishtank_match_last_url(self):
phishtank_inst = self.w3afcore.plugins.get_plugin_inst('crawl',
'phishtank')
vuln_url = URL(self.get_last_vulnerable_url())
phishtank_inst.crawl(FuzzableRequest(vuln_url))
vulns = self.kb.get('phishtank', 'phishtank')
self.assertEqual(len(vulns), 1, vulns)
vuln = vulns[0]
self.assertEqual(vuln.get_name(), 'Phishing scam')
self.assertEqual(vuln.get_severity(), MEDIUM)
self.assertEqual(vuln.get_url().get_domain(), vuln_url.get_domain())
def do_ALL(self):
global global_first_request
if global_first_request:
global_first_request = False
msg = 'The user is navigating through the spider_man proxy.'
om.out.information(msg)
# convert relative URL to absolute if request came from CONNECT
if hasattr(self.server, 'chainedHandler'):
base_path = "https://" + self.server.chainedHandler.path
path = base_path + self.path
else:
path = self.path
# Convert to url_object
path = URL(path)
# Ignore favicon.ico requests
# https://github.com/andresriancho/w3af/issues/9135
if path == TERMINATE_FAVICON_URL:
return
if path == TERMINATE_URL:
om.out.information('The user terminated the spider_man session.')
self._send_end()
self._spider_man.stop_proxy()
return
msg = '[spider_man] Handling request: %s %s'
om.out.debug(msg % (self.command, path))
# Send this information to the plugin so it can send it to the core
freq = self._create_fuzzable_request()
self._spider_man.append_fuzzable_request(freq)
grep = True
if path.get_domain() != self.server.w3afLayer.target_domain:
grep = False
try:
response = self._send_to_server(grep=grep)
except Exception, e:
self._send_error(e)
def do_ALL(self):
global global_first_request
if global_first_request:
global_first_request = False
msg = 'The user is navigating through the spider_man proxy.'
om.out.information(msg)
# convert relative URL to absolute if request came from CONNECT
if hasattr(self.server, 'chainedHandler'):
base_path = "https://" + self.server.chainedHandler.path
path = base_path + self.path
else:
path = self.path
# Convert to url_object
path = URL(path)
# Ignore favicon.ico requests
# https://github.com/andresriancho/w3af/issues/9135
if path == TERMINATE_FAVICON_URL:
return
if path == TERMINATE_URL:
om.out.information('The user terminated the spider_man session.')
self._send_end()
self._spider_man.stop_proxy()
return
msg = '[spider_man] Handling request: %s %s'
om.out.debug(msg % (self.command, path))
# Send this information to the plugin so it can send it to the core
freq = self._create_fuzzable_request()
self._spider_man.append_fuzzable_request(freq)
grep = True
if path.get_domain() != self.server.w3afLayer.target_domain:
grep = False
try:
response = self._send_to_server(grep=grep)
except Exception, e:
self._send_error(e)
def setUp(self):
self.kb.cleanup()
self.w3afcore = w3afCore()
if self.MOCK_RESPONSES:
httpretty.enable()
try:
url = URL(self.target_url)
except ValueError, ve:
msg = 'When using MOCK_RESPONSES you need to set the'\
' target_url attribute to a valid URL, exception was:'\
' "%s".'
raise Exception(msg % ve)
domain = url.get_domain()
proto = url.get_protocol()
port = url.get_port()
self._register_httpretty_uri(proto, domain, port)
def __init__(self, url, body, content_type='text/html', status=200,
method='GET', headers=None, delay=None):
self.url = url
self.body = body
self.status = status
self.method = method
self.delay = delay
self.content_type = content_type
self.headers = {'Content-Type': content_type}
if headers is not None:
self.headers.update(headers)
assert method in self.KNOWN_METHODS, self.NO_MOCK
assert isinstance(url, (basestring, RE_COMPILE_TYPE))
if isinstance(url, basestring):
url = URL(url)
assert url.get_domain(), 'Need to specify the MockResponse domain'
def test_set_domain_with_port(self):
u = URL('http://w3af.com:443/def/jkl/')
self.assertEqual(u.get_domain(), 'w3af.com')
u.set_domain('host.tld')
self.assertEqual(u.get_net_location(), 'host.tld:443')
def alert_if_target_is_301_all(self):
"""
Alert the user when the configured target is set to a site which will
301 redirect all requests to https://
:see: https://github.com/andresriancho/w3af/issues/14976
:return: True if the site returns 301 for all resources. Also an Info
instance is saved to the KB in order to alert the user.
"""
site_does_redirect = False
msg = ('The configured target domain redirects all HTTP requests to a'
' different location. The most common scenarios are:\n\n'
''
' * HTTP redirect to HTTPS\n'
' * domain.com redirect to www.domain.com\n\n'
''
'While the scan engine can identify URLs and vulnerabilities'
' using the current configuration it might be wise to start'
' a new scan setting the target URL to the redirect target.')
targets = cf.cf.get('targets')
for url in targets:
# We test if the target URLs are redirecting to a different protocol
# or domain.
try:
http_response = self._w3af_core.uri_opener.GET(url, cache=False)
except ScanMustStopByUserRequest:
# Not a real error, the user stopped the scan
raise
except Exception, e:
emsg = 'Exception found during alert_if_target_is_301_all(): "%s"'
emsg %= e
om.out.debug(emsg)
raise ScanMustStopException(emsg)
else:
if 300 <= http_response.get_code() <= 399:
# Get the redirect target
lower_headers = http_response.get_lower_case_headers()
redirect_url = None
for header_name in ('location', 'uri'):
if header_name in lower_headers:
header_value = lower_headers[header_name]
header_value = header_value.strip()
try:
redirect_url = URL(header_value)
except ValueError:
# No special invalid URL handling required
continue
if not redirect_url:
continue
# Check if the protocol was changed:
target_proto = url.get_protocol()
redirect_proto = redirect_url.get_protocol()
if target_proto != redirect_proto:
site_does_redirect = True
break
# Check if the domain was changed:
target_domain = url.get_domain()
redirect_domain = redirect_url.get_domain()
if target_domain != redirect_domain:
site_does_redirect = True
break
def alert_if_target_is_301_all(self):
"""
Alert the user when the configured target is set to a site which will
301 redirect all requests to https://
:see: https://github.com/andresriancho/w3af/issues/14976
:return: True if the site returns 301 for all resources. Also an Info
instance is saved to the KB in order to alert the user.
"""
site_does_redirect = False
msg = ('The configured target domain redirects all HTTP requests to a'
' different location. The most common scenarios are:\n\n'
''
' * HTTP redirect to HTTPS\n'
' * domain.com redirect to www.domain.com\n\n'
''
'While the scan engine can identify URLs and vulnerabilities'
' using the current configuration it might be wise to start'
' a new scan setting the target URL to the redirect target.')
targets = cf.cf.get('targets')
for url in targets:
# We test if the target URLs are redirecting to a different protocol
# or domain.
try:
http_response = self._w3af_core.uri_opener.GET(url,
cache=False)
except ScanMustStopByUserRequest:
# Not a real error, the user stopped the scan
raise
except Exception, e:
emsg = 'Exception found during alert_if_target_is_301_all(): "%s"'
emsg %= e
om.out.debug(emsg)
raise ScanMustStopException(emsg)
else:
if 300 <= http_response.get_code() <= 399:
# Get the redirect target
lower_headers = http_response.get_lower_case_headers()
redirect_url = None
for header_name in ('location', 'uri'):
if header_name in lower_headers:
header_value = lower_headers[header_name]
header_value = header_value.strip()
try:
redirect_url = URL(header_value)
except ValueError:
# No special invalid URL handling required
continue
if not redirect_url:
continue
# Check if the protocol was changed:
target_proto = url.get_protocol()
redirect_proto = redirect_url.get_protocol()
if target_proto != redirect_proto:
site_does_redirect = True
break
# Check if the domain was changed:
target_domain = url.get_domain()
redirect_domain = redirect_url.get_domain()
if target_domain != redirect_domain:
site_does_redirect = True
break
def test_set_domain_with_port(self):
u = URL('http://w3af.com:443/def/jkl/')
self.assertEqual(u.get_domain(), 'w3af.com')
u.set_domain('host.tld')
self.assertEqual(u.get_net_location(), 'host.tld:443')
