def test_low_level_with_cookie_jar(self):
# IMPORTANT NOTE: Please remember that the cookie expiration, 2736616305
# above, is going to limit the date until which this unittest will PASS
cj_contents = self.COOKIEJAR.replace(' ' * 8, '')
tmp_file = tempfile.NamedTemporaryFile(delete=False)
tmp_file.write(cj_contents)
tmp_file.close()
cj = cookielib.MozillaCookieJar()
cj.load(tmp_file.name, ignore_discard=True, ignore_expires=True)
cookie_handler = CookieHandler(cj)
opener = urllib2.build_opener(cookie_handler)
# Verify cookie from cookie jar is sent
with_cookie_req = HTTPRequest(self.URL_CHECK_COOKIE, cookies=True)
with_cookie_res = opener.open(with_cookie_req).read()
self.assertTrue('Cookie was sent.' in with_cookie_res)
# And now it will NOT send any cookie because we're setting cookie to False
no_cookie_req = HTTPRequest(self.URL_CHECK_COOKIE, cookies=False)
no_cookie_res = opener.open(no_cookie_req).read()
self.assertTrue('Cookie was NOT sent.' in no_cookie_res)
os.unlink(tmp_file.name)
def setUp(self):
super(TestHTMLRendering, self).setUp()
self.plugin = self.w3afcore.plugins.get_plugin_inst(
'output', 'html_file')
HistoryItem().init()
url = URL('http://w3af.com/a/b/c.php')
request = HTTPRequest(url, data='a=1')
hdr = Headers([('Content-Type', 'text/html')])
res = HTTPResponse(200, '<html>', hdr, url, url)
h1 = HistoryItem()
h1.request = request
res.set_id(1)
h1.response = res
h1.save()
url = URL('http://w3af.com/foo.py')
request = HTTPRequest(url, data='text=xss')
hdr = Headers([('Content-Type', 'text/html')])
res = HTTPResponse(200, '<html>empty</html>', hdr, url, url)
h1 = HistoryItem()
h1.request = request
res.set_id(4)
h1.response = res
h1.save()
def test_low_level(self):
httpretty.register_uri(
httpretty.GET,
self.URL_SEND_COOKIE,
adding_headers={'Set-Cookie': self.COOKIE_VALUE})
httpretty.register_uri(httpretty.GET,
self.URL_CHECK_COOKIE,
body=self.request_callback)
opener = urllib2.build_opener(CookieHandler)
# With this request the CookieHandler should store a cookie in its
# cookiejar
set_cookie_req = HTTPRequest(URL(self.URL_SEND_COOKIE))
opener.open(set_cookie_req).read()
# And now it will send it because we're setting cookie to True
with_cookie_req = HTTPRequest(URL(self.URL_CHECK_COOKIE), cookies=True)
with_cookie_res = opener.open(with_cookie_req).read()
self.assertIn('Cookie received', with_cookie_res)
# And now it will NOT send any cookie because we're setting cookie to False
no_cookie_req = HTTPRequest(URL(self.URL_CHECK_COOKIE), cookies=False)
no_cookie_res = opener.open(no_cookie_req).read()
self.assertIn('Cookie not sent', no_cookie_res)
# And now it will send it because we're setting cookie to True
with_cookie_req = HTTPRequest(URL(self.URL_CHECK_COOKIE), cookies=True)
with_cookie_res = opener.open(with_cookie_req).read()
self.assertIn('Cookie received', with_cookie_res)
def modify_request(self, request):
"""
Mangles the request
:param request: HTTPRequest instance that is going to be modified
by the evasion plugin
:return: The modified request
"""
# First we mangle the URL
path = request.url_object.get_path()
path = self._mutate(path)
# Finally, we set all the mutants to the request in order to return it
new_url = request.url_object.copy()
new_url.set_path(path)
# Mangle the postdata
data = request.get_data()
if data:
try:
# Only mangle the postdata if it is a url encoded string
parse_qs(data)
except:
pass
else:
data = self._mutate(data)
new_req = HTTPRequest(new_url,
data,
request.headers,
request.get_origin_req_host(),
retries=request.retries_left)
return new_req
def test_add_when_qs(self):
url = URL('http://www.w3af.com/?id=1')
original_req = HTTPRequest(url)
modified_req = self.eplugin.modify_request(original_req)
self.assertEqual(len(modified_req.url_object.querystring), 2)
self.assertIn('id=1', str(modified_req.url_object.querystring))
def test_no_modification(self):
modsec = mod_security()
u = URL('http://www.w3af.com/')
r = HTTPRequest( u )
self.assertEqual(modsec.modify_request( r ).url_object.url_string,
u'http://www.w3af.com/')
def create_redirect_request(self, req, fp, code, msg, headers, new_url_str,
new_url_obj):
"""
This is called by the http_error_30x methods when a
redirection response is received. If a redirection should
take place, return a new Request to allow http_error_30x to
perform the redirect.
"""
new_headers = dict((k, v) for k, v in req.headers.items()
if k.lower() not in REMOVE_ON_REDIRECT)
orig_method = req.get_method()
method = orig_method if orig_method in {'GET', 'HEAD'} else 'GET'
new_request = HTTPRequest(new_url_obj,
headers=new_headers,
origin_req_host=req.get_origin_req_host(),
method=method,
timeout=req.timeout,
unverifiable=True,
follow_redirects=True,
cookies=req.cookies,
cache=req.get_from_cache,
error_handling=req.error_handling,
retries=req.retries_left,
new_connection=req.new_connection,
use_basic_auth=req.use_basic_auth)
return new_request
def test_render_attr_with_special_chars(self):
_id = 2
name = 'A long description with special characters: <&">'
vuln = MockVuln(_id=_id)
vuln.set_name(name)
url = URL('http://w3af.com/a/b/c.php')
hdr = Headers([('User-Agent', 'w3af')])
request = HTTPRequest(url, data='a=1')
request.set_headers(hdr)
hdr = Headers([('Content-Type', 'text/html')])
res = HTTPResponse(200, '<html>', hdr, url, url)
h1 = HistoryItem()
h1.request = request
res.set_id(_id)
h1.response = res
h1.save()
x = xml_file()
finding = Finding(x._get_jinja2_env(), vuln)
xml = finding.to_string()
self.assertNotIn(name, xml)
self.assertIn(
'A long description with special characters: <&">',
xml)
self.assertValidXML(xml)
def test_render_with_special_chars(self):
_id = 2
desc = ('This is a long description that contains some special'
' characters such as <, & and > which MUST be encoded'
' by jinja2.')
vuln = MockVuln(_id=_id)
vuln.set_desc(desc)
url = URL('http://w3af.com/a/b/c.php')
hdr = Headers([('User-Agent', 'w3af')])
request = HTTPRequest(url, data='a=1')
request.set_headers(hdr)
hdr = Headers([('Content-Type', 'text/html')])
res = HTTPResponse(200, '<html>', hdr, url, url)
h1 = HistoryItem()
h1.request = request
res.set_id(_id)
h1.response = res
h1.save()
x = xml_file()
finding = Finding(x._get_jinja2_env(), vuln)
xml = finding.to_string()
self.assertNotIn('such as <, & and > which MUST', xml)
self.assertIn('such as <, & and > which MUST', xml)
self.assertValidXML(xml)
def test_add_when_qs(self):
rp = rnd_param()
u = URL('http://www.w3af.com/?id=1')
r = HTTPRequest(u)
qs = rp.modify_request(r).url_object.querystring
self.assertEqual(len(qs), 2)
def test_no_modification(self):
rhe = rnd_hex_encode()
u = URL('http://www.w3af.com/')
r = HTTPRequest( u )
self.assertEqual(rhe.modify_request( r ).url_object.url_string,
u'http://www.w3af.com/')
def any_method(uri_opener, method, uri, data=None, headers=Headers(),
cache=False, grep=True, cookies=True,
error_handling=True, timeout=None, use_basic_auth=True,
use_proxy=True, follow_redirects=False):
"""
:return: An HTTPResponse object that's the result of sending
the request with a method different from GET or POST.
"""
if not isinstance(uri, URL):
raise TypeError('The uri parameter of any_method must be'
' of url.URL type.')
if not isinstance(headers, Headers):
raise TypeError('The headers parameter of any_method must be'
' of Headers type.')
uri_opener.setup()
max_retries = uri_opener.settings.get_max_retrys()
new_connection = True if timeout is not None else False
host = uri.get_domain()
timeout = uri_opener.get_timeout(host) if timeout is None else timeout
req = HTTPRequest(uri, data, cookies=cookies, cache=cache,
method=method,
error_handling=error_handling,
retries=max_retries,
timeout=timeout,
new_connection=new_connection,
use_basic_auth=use_basic_auth,
follow_redirects=follow_redirects,
use_proxy=True)
req = uri_opener.add_headers(req, headers or {})
return uri_opener.send(req, grep=grep)
def modify_request(self, request):
"""
Mangles the request
:param request: HTTPRequest instance that is going to be modified by
the evasion plugin
:return: The modified request
"""
# We mangle the URL
path = request.url_object.get_path()
if re.match('^/', path):
random_alnum = rand_alnum()
path = '/' + random_alnum + '/..' + path
# Finally, we set all the mutants to the request in order to return it
new_url = request.url_object.copy()
new_url.set_path(path)
# Finally, we set all the mutants to the request in order to return it
new_req = HTTPRequest(new_url,
request.data,
request.headers,
request.get_origin_req_host(),
retries=request.retries_left)
return new_req
def test_path_file(self):
rs = reversed_slashes()
u = URL('http://www.w3af.com/abc/def.htm')
r = HTTPRequest( u )
self.assertEqual(rs.modify_request( r ).url_object.url_string,
u'http://www.w3af.com/abc\\def.htm')
def test_no_modification(self):
rs = reversed_slashes()
u = URL('http://www.w3af.com/')
r = HTTPRequest( u )
self.assertEqual(rs.modify_request( r ).url_object.url_string,
u'http://www.w3af.com/')
def test_modify_post_data(self):
rc = rnd_case()
u = URL('http://www.w3af.com/')
r = HTTPRequest(u, data='a=b')
modified_data = rc.modify_request(r).get_data()
self.assertIn(modified_data, ['a=b', 'A=b', 'a=B', 'A=B'])
def test_clear(self):
url = URL('http://w3af.com/a/b/c.php')
request = HTTPRequest(url, data='a=1')
hdr = Headers([('Content-Type', 'text/html')])
res = HTTPResponse(200, '<html>', hdr, url, url)
h1 = HistoryItem()
h1.request = request
res.set_id(1)
h1.response = res
h1.save()
table_name = h1.get_table_name()
db = get_default_temp_db_instance()
self.assertTrue(db.table_exists(table_name))
clear_result = h1.clear()
self.assertTrue(clear_result)
self.assertFalse(os.path.exists(h1._session_dir),
'%s exists.' % h1._session_dir)
# Changed the meaning of clear a little bit... now it simply removes
# all rows from the table, not the table itself
self.assertTrue(db.table_exists(table_name))
def _server_root_path_is_reachable(self, request):
"""
Sends an HTTP GET to the server's root path to verify that it's
reachable from our location.
:param request: The original HTTP request
:return: True if we were able to get a response
"""
uri = request.get_uri()
root_url = uri.base_url()
host = uri.get_domain()
req = HTTPRequest(root_url,
cookies=True,
cache=False,
error_handling=False,
method='GET',
retries=0,
timeout=self.get_timeout(host))
req = self.add_headers(req)
try:
self.send(req, grep=False)
except HTTPRequestException, e:
msg = 'Remote URL %s is UNREACHABLE due to: "%s"'
om.out.debug(msg % (root_url, e))
return False
def test_modify_basic(self):
bbd = backspace_between_dots()
u = URL('http://www.w3af.com/../')
r = HTTPRequest(u)
self.assertEqual(bbd.modify_request(r).url_object.url_string,
u'http://www.w3af.com/.%41%08./')
def test_handler_integration(self):
"""
Integration test with w3af's URL opener.
"""
test_param = 'test_handler_integration'
settings = opener_settings.OpenerSettings()
settings.set_url_parameter(test_param)
settings.build_openers()
opener = settings.get_custom_opener()
for proto in {'http', 'https'}:
test_url = URL('%s://mock/abc/def.html' % proto)
test_url_param = URL('%s://mock/abc/def.html;%s' %
(proto, test_param))
request = HTTPRequest(test_url)
httpretty.register_uri(httpretty.GET,
test_url.url_string,
body='FAIL')
httpretty.register_uri(httpretty.GET,
test_url_param.url_string,
body='SUCCESS')
response = opener.open(request)
self.assertIn('SUCCESS', response.read())
def test_encode_path_case01(self):
rhe = rnd_hex_encode()
u = URL('http://www.w3af.com/a/')
r = HTTPRequest( u )
modified_path = rhe.modify_request( r ).url_object.get_path()
self.assertIn(modified_path, ['/a/','/%61/'])
def create_redirect_request(self, request, response, new_url_obj):
"""
Create a new HTTP request inheriting all the attributes from the
original object and setting the target URL to the one received in the
30x response.
"""
new_headers = dict((k, v) for k, v in request.headers.items()
if k.lower() not in REMOVE_ON_REDIRECT)
orig_method = request.get_method()
method = orig_method if orig_method in GET_HEAD else 'GET'
new_request = HTTPRequest(
new_url_obj,
headers=new_headers,
origin_req_host=request.get_origin_req_host(),
method=method,
timeout=request.timeout,
unverifiable=True,
follow_redirects=True,
cookies=request.cookies,
cache=request.get_from_cache,
error_handling=request.error_handling,
retries=request.retries_left,
new_connection=request.new_connection,
use_basic_auth=request.use_basic_auth)
return new_request
def GET(self,
uri,
data=None,
headers=Headers(),
cache=False,
grep=True,
cookies=True,
respect_size_limit=True,
error_handling=True,
timeout=None):
"""
HTTP GET a URI using a proxy, user agent, and other settings
that where previously set in opener_settings.py .
:param uri: This is the URI to GET, with the query string included.
:param data: Only used if the uri parameter is really a URL. The data
will be converted into a string and set as the URL object
query string before sending.
:param headers: Any special headers that will be sent with this request
:param cache: Should the library search the local cache for a response
before sending it to the wire?
:param grep: Should grep plugins be applied to this request/response?
:param timeout: If None we'll use the configured (opener settings)
timeout or the auto-adjusted value. Otherwise we'll use
the defined timeout as the socket timeout value for this
request. The timeout is specified in seconds
:param cookies: Send stored cookies in request (or not)
:return: An HTTPResponse object.
"""
if not isinstance(uri, URL):
raise TypeError('The uri parameter of ExtendedUrllib.GET() must be'
' of url.URL type.')
if not isinstance(headers, Headers):
raise TypeError('The header parameter of ExtendedUrllib.GET() must'
' be of Headers type.')
# Validate what I'm sending, init the library (if needed)
self.setup()
if data:
uri = uri.copy()
uri.querystring = data
new_connection = True if timeout is not None else False
host = uri.get_domain()
timeout = self.get_timeout(host) if timeout is None else timeout
req = HTTPRequest(uri,
cookies=cookies,
cache=cache,
error_handling=error_handling,
method='GET',
retries=self.settings.get_max_retrys(),
timeout=timeout,
new_connection=new_connection)
req = self.add_headers(req, headers)
with raise_size_limit(respect_size_limit):
return self.send(req, grep=grep)
def modify_request(self, request):
"""
Mangles the request
:param request: HTTPRequest instance that is going to be modified
by the evasion plugin
:return: The modified request
"""
# First we mangle the URL
qs = request.url_object.querystring.copy()
qs = self._mutate(qs)
# Finally, we set all the mutants to the request in order to return it
new_url = request.url_object.copy()
new_url.querystring = qs
# Mangle the postdata
post_data = request.get_data()
if post_data:
try:
# Only mangle the postdata if it is a url encoded string
post_data = parse_qs(post_data)
except:
pass
else:
post_data = str(self._mutate(post_data))
new_req = HTTPRequest(new_url, post_data, request.headers,
request.get_origin_req_host())
return new_req
def test_render_with_unicode_control_chars(self):
_id = 2
desc = ('This is a long description that contains some special'
' unicode control characters such as \f and \x09')
vuln = MockVuln(_id=_id)
vuln.set_desc(desc)
url = URL('http://w3af.com/a/b/c.php')
hdr = Headers([('User-Agent', 'w3af')])
request = HTTPRequest(url, data='a=1')
request.set_headers(hdr)
hdr = Headers([('Content-Type', 'text/html')])
res = HTTPResponse(200, '<html>', hdr, url, url)
h1 = HistoryItem()
h1.request = request
res.set_id(_id)
h1.response = res
h1.save()
x = xml_file()
finding = Finding(x._get_jinja2_env(), vuln)
xml = finding.to_string()
self.assertNotIn('unicode control characters such as \f and \x09', xml)
self.assertIn(
'unicode control characters such as <character code="000c"/> and <character code="0009"/>',
xml)
self.assertValidXML(xml)
def test_cache_http_errors(self):
settings = opener_settings.OpenerSettings()
settings.build_openers()
opener = settings.get_custom_opener()
url = URL('http://w3af.org/foo-bar-not-exists.htm')
request = HTTPRequest(url, cache=False)
with patch('w3af.core.data.url.handlers.cache.CacheClass') as cc_mock:
store_in_cache = Mock()
cc_mock.attach_mock(store_in_cache, 'store_in_cache')
# If there is a response we should store it, even if it is a 404
try:
response = opener.open(request)
except urllib2.HTTPError:
pass
# Make sure the right call was made
_call = _Call(('store_in_cache', (request, response)))
self.assertEqual(cc_mock.mock_calls, [_call])
cc_mock.reset_mock()
# And make sure the response was a 404
self.assertEqual(response.status, 404)
def __call__(self,
uri,
data=None,
headers=Headers(),
cache=False,
grep=True,
cookies=True,
ignore_errors=False):
"""
:return: An HTTPResponse object that's the result of
sending the request with a method different from
"GET" or "POST".
"""
if not isinstance(uri, URL):
raise TypeError('The uri parameter of AnyMethod.'
'__call__() must be of url.URL type.')
if not isinstance(headers, Headers):
raise TypeError('The headers parameter of AnyMethod.'
'__call__() must be of Headers type.')
self._xurllib._init()
max_retries = self._xurllib.settings.get_max_retrys()
req = HTTPRequest(uri,
data,
cookies=cookies,
cache=cache,
method=self._method,
ignore_errors=ignore_errors,
retries=max_retries)
req = self._xurllib._add_headers(req, headers or {})
return self._xurllib._send(req, grep=grep)
def test_encode_post_data(self):
rhe = rnd_hex_encode()
u = URL('http://www.w3af.com/')
r = HTTPRequest(u, data='a=b')
modified_pdata = rhe.modify_request(r).get_data()
self.assertIn(modified_pdata, ['a=b', '%61=b', 'a=%62', '%61=%62'])
def modify_request(self, request):
"""
Mangles the request
:param request: HTTPRequest instance that is going to be modified by
the evasion plugin
:return: The modified request
"""
# Mangle the postdata
data = str(request.get_data())
if data:
try:
# Only mangle the postdata if it is a url encoded string
parse_qs(data)
except:
pass
else:
data = '\x00' + data
headers_copy = copy.deepcopy(request.headers)
headers_copy['content-length'] = str(len(data))
request = HTTPRequest(request.url_object,
data,
headers_copy,
request.get_origin_req_host(),
retries=request.retries_left)
return request
def test_find(self):
find_id = random.randint(1, 499)
url = URL('http://w3af.org/a/b/foobar.php?foo=123')
tag_value = rand_alnum(10)
for i in xrange(0, 500):
request = HTTPRequest(url, data='a=1')
code = 200
if i == find_id:
code = 302
hdr = Headers([('Content-Type', 'text/html')])
res = HTTPResponse(code, '<html>', hdr, url, url)
h1 = HistoryItem()
h1.request = request
res.set_id(i)
h1.response = res
if i == find_id:
h1.toggle_mark()
h1.update_tag(tag_value)
h1.save()
h2 = HistoryItem()
self.assertEqual(
len(h2.find([('tag', "%" + tag_value + "%", 'like')])), 1)
self.assertEqual(len(h2.find([('code', 302, '=')])), 1)
self.assertEqual(len(h2.find([('mark', 1, '=')])), 1)
self.assertEqual(len(h2.find([('has_qs', 1, '=')])), 500)
self.assertEqual(
len(h2.find([('has_qs', 1, '=')], result_limit=10)), 10)
results = h2.find(
[('has_qs', 1, '=')], result_limit=1, orderData=[('id', 'desc')])
self.assertEqual(results[0].id, 499)
search_data = []
search_data.append(('id', find_id + 1, "<"))
search_data.append(('id', find_id - 1, ">"))
self.assertEqual(len(h2.find(search_data)), 1)
