def modify_request(self, request):
'''
Mangles the request
:param request: HTTPRequest instance that is going to be modified by
the evasion plugin
:return: The modified request
'''
# Mangle the postdata
data = str(request.get_data())
if data:
try:
# Only mangle the postdata if it is a url encoded string
parse_qs(data)
except:
pass
else:
data = '\x00' + data
headers_copy = copy.deepcopy(request.headers)
headers_copy['content-length'] = str(len(data))
request = HTTPRequest(request.url_object, data, headers_copy,
request.get_origin_req_host())
return request
def modify_request(self, request):
'''
Mangles the request
:param request: HTTPRequest instance that is going to be modified by
the evasion plugin
:return: The modified request
'''
# Mangle the postdata
data = str(request.get_data())
if data:
try:
# Only mangle the postdata if it is a url encoded string
parse_qs(data)
except:
pass
else:
data = '\x00' + data
headers_copy = copy.deepcopy(request.headers)
headers_copy['content-length'] = str(len(data))
request = HTTPRequest(request.url_object, data, headers_copy,
request.get_origin_req_host())
return request
def test_low_level_with_cookie_jar(self):
# IMPORTANT NOTE: Please remember that the cookie expiration, 2736616305
# above, is going to limit the date until which this unittest will PASS
cj_contents = self.COOKIEJAR.replace(' ' * 8, '')
tmp_file = tempfile.NamedTemporaryFile(delete=False)
tmp_file.write(cj_contents)
tmp_file.close()
cj = cookielib.MozillaCookieJar()
cj.load(tmp_file.name, ignore_discard=True, ignore_expires=True)
cookie_handler = CookieHandler(cj)
opener = urllib2.build_opener(cookie_handler)
# Verify cookie from cookie jar is sent
with_cookie_req = HTTPRequest(self.URL_CHECK_COOKIE, cookies=True)
with_cookie_res = opener.open(with_cookie_req).read()
self.assertTrue('Cookie was sent.' in with_cookie_res)
# And now it will NOT send any cookie because we're setting cookie to False
no_cookie_req = HTTPRequest(self.URL_CHECK_COOKIE, cookies=False)
no_cookie_res = opener.open(no_cookie_req).read()
self.assertTrue('Cookie was NOT sent.' in no_cookie_res)
os.unlink(tmp_file.name)
def test_dump_case01(self):
expected = '\r\n'.join(
['GET http://w3af.com/a/b/c.php HTTP/1.1', 'Hello: World', '', ''])
u = URL('http://w3af.com/a/b/c.php')
headers = Headers([('Hello', 'World')])
req = HTTPRequest(u, headers=headers)
self.assertEqual(req.dump(), expected)
def test_basic(self):
u = URL('http://www.w3af.com')
req = HTTPRequest(u)
self.assertEqual(req.get_full_url(),
'http://www.w3af.com/')
self.assertEqual(req.get_uri().url_string,
'http://www.w3af.com/')
def test_dump_case02(self):
expected = u'\r\n'.join([
u'GET http://w3af.com/a/b/c.php HTTP/1.1', u'Hola: Múndo', u'', u''
])
u = URL('http://w3af.com/a/b/c.php')
headers = Headers([('Hola', 'Múndo')])
req = HTTPRequest(u, headers=headers)
self.assertEqual(req.dump(), expected)
def test_dump_case02(self):
expected = u'\r\n'.join([u'GET http://w3af.com/a/b/c.php HTTP/1.1',
u'Hola: Múndo',
u'',
u''])
u = URL('http://w3af.com/a/b/c.php')
headers = Headers([('Hola', 'Múndo')])
req = HTTPRequest(u, headers=headers)
self.assertEqual(req.dump(), expected)
def test_dump_case01(self):
expected = '\r\n'.join(['GET http://w3af.com/a/b/c.php HTTP/1.1',
'Hello: World',
'',
''])
u = URL('http://w3af.com/a/b/c.php')
headers = Headers([('Hello', 'World')])
req = HTTPRequest(u, headers=headers)
self.assertEqual(req.dump(), expected)
def test_to_from_dict(self):
headers = Headers([('Host', 'www.w3af.com')])
req = HTTPRequest(URL("http://www.w3af.com/"),
data='spameggs',
headers=headers)
msg = msgpack.dumps(req.to_dict())
loaded_dict = msgpack.loads(msg)
loaded_req = HTTPRequest.from_dict(loaded_dict)
self.assertEqual(req, loaded_req)
self.assertEqual(req.__dict__.values(), loaded_req.__dict__.values())
def test_to_from_dict(self):
headers = Headers([('Host', 'www.w3af.com')])
req = HTTPRequest(URL("http://www.w3af.com/"), data='spameggs',
headers=headers)
msg = msgpack.dumps(req.to_dict())
loaded_dict = msgpack.loads(msg)
loaded_req = HTTPRequest.from_dict(loaded_dict)
self.assertEqual(req, loaded_req)
self.assertEqual(req.__dict__.values(),
loaded_req.__dict__.values())
def _new_no_content_resp(self, uri, log_it=False):
'''
Return a new NO_CONTENT HTTPResponse object. Optionally call the
subscribed log handlers
:param uri: URI string or request object
:param log_it: Boolean that indicated whether to log request
and response.
'''
# accept a URI or a Request object
if isinstance(uri, URL):
req = HTTPRequest(uri)
elif isinstance(uri, HTTPRequest):
req = uri
else:
msg = 'The uri parameter of ExtendedUrllib._new_content_resp() has to be'\
' of HTTPRequest of URL type.'
raise Exception(msg)
# Work,
no_content_response = HTTPResponse(NO_CONTENT, '', Headers(), uri,
uri, msg='No Content')
if log_it:
# This also assigns the id to both objects.
LogHandler.log_req_resp(req, no_content_response)
if no_content_response.id is None:
no_content_response.id = seq_gen.inc()
return no_content_response
def _load_from_file(self, id):
fname = self._get_fname_for_id(id)
#
# Due to some concurrency issues, we need to perform this check
# before we try to read the .trace file.
#
if not os.path.exists(fname):
for _ in xrange(1 / 0.05):
time.sleep(0.05)
if os.path.exists(fname):
break
else:
msg = 'Timeout expecting trace file to be written "%s"' % fname
raise IOError(msg)
#
# Ok... the file exists, but it might still be being written
#
req_res = open(fname, 'rb')
request_dict, response_dict = msgpack.load(req_res)
req_res.close()
request = HTTPRequest.from_dict(request_dict)
response = HTTPResponse.from_dict(response_dict)
return (request, response)
def modify_request(self, request):
'''
Mangles the request
:param request: HTTPRequest instance that is going to be modified
by the evasion plugin
:return: The modified request
'''
# First we mangle the URL
qs = request.url_object.querystring.copy()
qs = self._mutate(qs)
# Finally, we set all the mutants to the request in order to return it
new_url = request.url_object.copy()
new_url.querystring = qs
# Mangle the postdata
post_data = request.get_data()
if post_data:
try:
# Only mangle the postdata if it is a url encoded string
post_data = parse_qs(post_data)
except:
pass
else:
post_data = str(self._mutate(post_data))
new_req = HTTPRequest(new_url, post_data, request.headers,
request.get_origin_req_host())
return new_req
def test_modify_post_data(self):
rc = rnd_case()
u = URL('http://www.w3af.com/')
r = HTTPRequest(u, data='a=b')
modified_data = rc.modify_request(r).get_data()
self.assertIn(modified_data, ['a=b', 'A=b', 'a=B', 'A=B'])
def test_encode_post_data(self):
rhe = rnd_hex_encode()
u = URL('http://www.w3af.com/')
r = HTTPRequest(u, data='a=b')
modified_pdata = rhe.modify_request(r).get_data()
self.assertIn(modified_pdata, ['a=b', '%61=b', 'a=%62', '%61=%62'])
def test_add_when_qs(self):
rp = rnd_param()
u = URL('http://www.w3af.com/?id=1')
r = HTTPRequest(u)
qs = rp.modify_request(r).url_object.querystring
self.assertEqual(len(qs), 2)
def test_path_file(self):
rs = reversed_slashes()
u = URL('http://www.w3af.com/abc/def.htm')
r = HTTPRequest( u )
self.assertEqual(rs.modify_request( r ).url_object.url_string,
u'http://www.w3af.com/abc\\def.htm')
def _load_from_file(self, id):
fname = self._get_fname_for_id(id)
#
# Due to some concurrency issues, we need to perform this check
# before we try to read the .trace file.
#
if not os.path.exists(fname):
for _ in xrange(1 / 0.05):
time.sleep(0.05)
if os.path.exists(fname):
break
else:
msg = 'Timeout expecting trace file to be written "%s"' % fname
raise IOError(msg)
#
# Ok... the file exists, but it might still be being written
#
req_res = open(fname, 'rb')
request_dict, response_dict = msgpack.load(req_res)
req_res.close()
request = HTTPRequest.from_dict(request_dict)
response = HTTPResponse.from_dict(response_dict)
return (request, response)
def test_no_modification(self):
rs = reversed_slashes()
u = URL('http://www.w3af.com/')
r = HTTPRequest( u )
self.assertEqual(rs.modify_request( r ).url_object.url_string,
u'http://www.w3af.com/')
def __call__(self,
uri,
data=None,
headers=Headers(),
cache=False,
grep=True,
cookies=True):
'''
:return: An HTTPResponse object that's the result of
sending the request with a method different from
"GET" or "POST".
'''
if not isinstance(uri, URL):
raise TypeError('The uri parameter of AnyMethod.'
'__call__() must be of url.URL type.')
if not isinstance(headers, Headers):
raise TypeError('The headers parameter of AnyMethod.'
'__call__() must be of Headers type.')
self._xurllib._init()
req = HTTPRequest(uri,
data,
cookies=cookies,
cache=cache,
method=self._method)
req = self._xurllib._add_headers(req, headers or {})
return self._xurllib._send(req, grep=grep)
def test_encode_path_case01(self):
rhe = rnd_hex_encode()
u = URL('http://www.w3af.com/a/')
r = HTTPRequest(u)
modified_path = rhe.modify_request(r).url_object.get_path()
self.assertIn(modified_path, ['/a/', '/%61/'])
def test_add_with_filename(self):
rp = rnd_path()
u = URL('http://www.w3af.com/abc/def.htm')
r = HTTPRequest( u )
url_string = rp.modify_request( r ).url_object.url_string
self.assertRegexpMatches(url_string, 'http://www.w3af.com/\w*/../abc/def.htm')
def test_add_when_dotdot(self):
sosibd = shift_out_in_between_dots()
u = URL('http://www.w3af.com/../')
r = HTTPRequest( u )
self.assertEqual(sosibd.modify_request( r ).url_object.url_string,
u'http://www.w3af.com/.%0E%0F./')
def test_no_modification(self):
rhe = rnd_hex_encode()
u = URL('http://www.w3af.com/')
r = HTTPRequest(u)
self.assertEqual(
rhe.modify_request(r).url_object.url_string,
u'http://www.w3af.com/')
def test_no_modification(self):
modsec = mod_security()
u = URL('http://www.w3af.com/')
r = HTTPRequest(u)
self.assertEqual(
modsec.modify_request(r).url_object.url_string,
u'http://www.w3af.com/')
def test_add_path_to_base_url(self):
rp = rnd_path()
u = URL('http://www.w3af.com/')
r = HTTPRequest( u )
url_string = rp.modify_request( r ).url_object.url_string
self.assertRegexpMatches(url_string, 'http://www.w3af.com/\w*/../')
def test_no_modification(self):
sosibd = shift_out_in_between_dots()
u = URL('http://www.w3af.com/')
r = HTTPRequest( u )
self.assertEqual(sosibd.modify_request( r ).url_object.url_string,
u'http://www.w3af.com/')
def test_no_modification(self):
fwe = full_width_encode()
u = URL('http://www.w3af.com/')
r = HTTPRequest(u)
self.assertEqual(
fwe.modify_request(r).url_object.url_string,
u'http://www.w3af.com/')
def test_modify_path(self):
rc = rnd_case()
u = URL('http://www.w3af.com/ab/')
r = HTTPRequest(u)
modified_path = rc.modify_request(r).url_object.get_path()
self.assertIn(modified_path, ['/ab/', '/aB/', '/Ab/', '/AB/'])
def test_modify_basic(self):
bbd = backspace_between_dots()
u = URL('http://www.w3af.com/../')
r = HTTPRequest(u)
self.assertEqual(
bbd.modify_request(r).url_object.url_string,
u'http://www.w3af.com/.%41%08./')
def test_add_to_url_with_path(self):
sr = self_reference()
u = URL('http://www.w3af.com/abc/')
r = HTTPRequest(u)
self.assertEqual(
sr.modify_request(r).url_object.url_string,
u'http://www.w3af.com/./abc/./')
def http_request(self, req):
url_instance = URL(req.get_full_url())
url_instance.set_param(self._url_parameter)
new_request = HTTPRequest(url_instance,
headers=req.headers,
origin_req_host=req.get_origin_req_host(),
unverifiable=req.is_unverifiable())
return new_request
def test_from_HTTPRequest_headers(self):
hdr = Headers([('Foo', 'bar')])
request = HTTPRequest(self.url, headers=hdr)
fr = create_fuzzable_request_from_request(request)
self.assertEqual(fr.get_url(), self.url)
self.assertEqual(fr.get_headers(), hdr)
self.assertEqual(fr.get_method(), 'GET')
self.assertIsInstance(fr, HTTPQSRequest)
def modifyRequest(self, request ):
'''
Mangles the request
@parameter request: HTTPRequest instance that is going to be modified by the evasion plugin
@return: The modified request
>>> from core.data.parsers.urlParser import url_object
>>> modsec = modsecurity()
>>> u = url_object('http://www.google.com/')
>>> r = HTTPRequest( u )
>>> modsec.modifyRequest( r ).url_object.url_string
'http://www.google.com/'
>>> u = url_object('http://www.google.com/')
>>> r = HTTPRequest( u, data='' )
>>> modsec.modifyRequest( r ).get_data()
''
>>> u = url_object('http://www.google.com/')
>>> r = HTTPRequest( u, data='a=b' )
>>> modsec.modifyRequest( r ).get_data()
'\\x00a=b'
'''
# Mangle the postdata
data = str(request.get_data())
if data:
try:
# Only mangle the postdata if it is a url encoded string
parse_qs( data )
except:
pass
else:
data = '\x00' + data
headers_copy = copy.deepcopy(request.headers)
headers_copy['content-length'] = str(len(data))
request = HTTPRequest( request.url_object, data, headers_copy,
request.get_origin_req_host() )
return request
def test_handler_order_block(self):
'''Get an instance of the extended urllib and verify that the blacklist
handler still works, even when mixed with all the other handlers.'''
# Configure the handler
blocked_url = URL('http://moth/abc/def/')
cf.cf.save('non_targets', [blocked_url,])
settings = opener_settings.OpenerSettings()
settings.build_openers()
opener = settings.get_custom_opener()
request = HTTPRequest(blocked_url)
request.url_object = blocked_url
request.cookies = True
request.get_from_cache = False
response = opener.open(request)
self.assertEqual(response.code, NO_CONTENT)
self.assertEqual(response.id, 1)
def get_headers(self, uri):
'''
:param uri: The URI we want to know the request headers
:return: A Headers object with the HTTP headers that would be added by
the library when sending a request to uri.
'''
req = HTTPRequest(uri)
req = self._add_headers(req)
return Headers(req.headers)
def test_no_cache(self):
url = URL('http://www.w3af.org')
request = HTTPRequest(url, cache=False)
cache = CacheHandler()
self.assertEqual(cache.default_open(request), None)
response = FakeHttplibHTTPResponse(200, 'OK', 'spameggs', Headers(),
url.url_string)
cache.http_response(request, response)
self.assertEqual(cache.default_open(request), None)
def test_low_level(self):
opener = urllib2.build_opener(CookieHandler)
# With this request the CookieHandler should store a cookie in its
# cookiejar
set_cookie_req = HTTPRequest(self.URL_SENDS_COOKIE)
opener.open(set_cookie_req).read()
# And now it will send it because we're setting cookie to True
with_cookie_req = HTTPRequest(self.URL_CHECK_COOKIE, cookies=True)
with_cookie_res = opener.open(with_cookie_req).read()
self.assertTrue('Cookie was sent.' in with_cookie_res)
# And now it will NOT send any cookie because we're setting cookie to False
no_cookie_req = HTTPRequest(self.URL_CHECK_COOKIE, cookies=False)
no_cookie_res = opener.open(no_cookie_req).read()
self.assertTrue('Cookie was NOT sent.' in no_cookie_res)
# And now it will send it because we're setting cookie to True
with_cookie_req = HTTPRequest(self.URL_CHECK_COOKIE, cookies=True)
with_cookie_res = opener.open(with_cookie_req).read()
self.assertTrue('Cookie was sent.' in with_cookie_res)
def setUp(self):
create_temp_dir()
self.plugin = sed()
self.url = URL('http://www.w3af.com/')
self.request = HTTPRequest(self.url)
class TestSed(unittest.TestCase):
def setUp(self):
create_temp_dir()
self.plugin = sed()
self.url = URL('http://www.w3af.com/')
self.request = HTTPRequest(self.url)
def tearDown(self):
self.plugin.end()
def test_blank_body(self):
body = ''
headers = Headers([('content-type', 'text/html')])
response = HTTPResponse(200, body, headers, self.url, self.url, _id=1)
option_list = self.plugin.get_options()
option_list['expressions'].set_value('qh/User/NotLuser/')
self.plugin.set_options(option_list)
mod_request = self.plugin.mangle_request(self.request)
mod_response = self.plugin.mangle_response(response)
self.assertEqual(mod_request.get_headers(), self.request.get_headers())
self.assertEqual(mod_response.get_headers(), response.get_headers())
self.assertEqual(mod_request.get_uri(), self.request.get_uri())
self.assertEqual(mod_response.get_uri(), response.get_uri())
self.assertEqual(mod_response.get_body(), response.get_body())
def test_response_body(self):
body = 'hello user!'
headers = Headers([('content-type', 'text/html')])
response = HTTPResponse(200, body, headers, self.url, self.url, _id=1)
option_list = self.plugin.get_options()
option_list['expressions'].set_value('sb/user/notluser/')
self.plugin.set_options(option_list)
mod_request = self.plugin.mangle_request(self.request)
mod_response = self.plugin.mangle_response(response)
self.assertEqual(mod_request.get_headers(), self.request.get_headers())
self.assertEqual(mod_response.get_headers(), response.get_headers())
self.assertEqual(mod_request.get_uri(), self.request.get_uri())
self.assertEqual(mod_response.get_uri(), response.get_uri())
self.assertEqual(mod_response.get_body(), 'hello notluser!')
def test_request_headers(self):
headers = Headers([('content-type', 'text/html')])
request = HTTPRequest(self.url, headers=headers)
option_list = self.plugin.get_options()
option_list['expressions'].set_value('qh/html/xml/')
self.plugin.set_options(option_list)
mod_request = self.plugin.mangle_request(request)
value, _ = mod_request.get_headers().iget('content-type')
self.assertEqual(value, 'text/xml')