def modify_request(self, request): ''' Mangles the request :param request: HTTPRequest instance that is going to be modified by the evasion plugin :return: The modified request ''' # Mangle the postdata data = str(request.get_data()) if data: try: # Only mangle the postdata if it is a url encoded string parse_qs(data) except: pass else: data = '\x00' + data headers_copy = copy.deepcopy(request.headers) headers_copy['content-length'] = str(len(data)) request = HTTPRequest(request.url_object, data, headers_copy, request.get_origin_req_host()) return request
def test_low_level_with_cookie_jar(self): # IMPORTANT NOTE: Please remember that the cookie expiration, 2736616305 # above, is going to limit the date until which this unittest will PASS cj_contents = self.COOKIEJAR.replace(' ' * 8, '') tmp_file = tempfile.NamedTemporaryFile(delete=False) tmp_file.write(cj_contents) tmp_file.close() cj = cookielib.MozillaCookieJar() cj.load(tmp_file.name, ignore_discard=True, ignore_expires=True) cookie_handler = CookieHandler(cj) opener = urllib2.build_opener(cookie_handler) # Verify cookie from cookie jar is sent with_cookie_req = HTTPRequest(self.URL_CHECK_COOKIE, cookies=True) with_cookie_res = opener.open(with_cookie_req).read() self.assertTrue('Cookie was sent.' in with_cookie_res) # And now it will NOT send any cookie because we're setting cookie to False no_cookie_req = HTTPRequest(self.URL_CHECK_COOKIE, cookies=False) no_cookie_res = opener.open(no_cookie_req).read() self.assertTrue('Cookie was NOT sent.' in no_cookie_res) os.unlink(tmp_file.name)
def test_dump_case01(self): expected = '\r\n'.join( ['GET http://w3af.com/a/b/c.php HTTP/1.1', 'Hello: World', '', '']) u = URL('http://w3af.com/a/b/c.php') headers = Headers([('Hello', 'World')]) req = HTTPRequest(u, headers=headers) self.assertEqual(req.dump(), expected)
def test_basic(self): u = URL('http://www.w3af.com') req = HTTPRequest(u) self.assertEqual(req.get_full_url(), 'http://www.w3af.com/') self.assertEqual(req.get_uri().url_string, 'http://www.w3af.com/')
def test_dump_case02(self): expected = u'\r\n'.join([ u'GET http://w3af.com/a/b/c.php HTTP/1.1', u'Hola: MĂșndo', u'', u'' ]) u = URL('http://w3af.com/a/b/c.php') headers = Headers([('Hola', 'MĂșndo')]) req = HTTPRequest(u, headers=headers) self.assertEqual(req.dump(), expected)
def test_dump_case02(self): expected = u'\r\n'.join([u'GET http://w3af.com/a/b/c.php HTTP/1.1', u'Hola: MĂșndo', u'', u'']) u = URL('http://w3af.com/a/b/c.php') headers = Headers([('Hola', 'MĂșndo')]) req = HTTPRequest(u, headers=headers) self.assertEqual(req.dump(), expected)
def test_dump_case01(self): expected = '\r\n'.join(['GET http://w3af.com/a/b/c.php HTTP/1.1', 'Hello: World', '', '']) u = URL('http://w3af.com/a/b/c.php') headers = Headers([('Hello', 'World')]) req = HTTPRequest(u, headers=headers) self.assertEqual(req.dump(), expected)
def test_to_from_dict(self): headers = Headers([('Host', 'www.w3af.com')]) req = HTTPRequest(URL("http://www.w3af.com/"), data='spameggs', headers=headers) msg = msgpack.dumps(req.to_dict()) loaded_dict = msgpack.loads(msg) loaded_req = HTTPRequest.from_dict(loaded_dict) self.assertEqual(req, loaded_req) self.assertEqual(req.__dict__.values(), loaded_req.__dict__.values())
def _new_no_content_resp(self, uri, log_it=False): ''' Return a new NO_CONTENT HTTPResponse object. Optionally call the subscribed log handlers :param uri: URI string or request object :param log_it: Boolean that indicated whether to log request and response. ''' # accept a URI or a Request object if isinstance(uri, URL): req = HTTPRequest(uri) elif isinstance(uri, HTTPRequest): req = uri else: msg = 'The uri parameter of ExtendedUrllib._new_content_resp() has to be'\ ' of HTTPRequest of URL type.' raise Exception(msg) # Work, no_content_response = HTTPResponse(NO_CONTENT, '', Headers(), uri, uri, msg='No Content') if log_it: # This also assigns the id to both objects. LogHandler.log_req_resp(req, no_content_response) if no_content_response.id is None: no_content_response.id = seq_gen.inc() return no_content_response
def _load_from_file(self, id): fname = self._get_fname_for_id(id) # # Due to some concurrency issues, we need to perform this check # before we try to read the .trace file. # if not os.path.exists(fname): for _ in xrange(1 / 0.05): time.sleep(0.05) if os.path.exists(fname): break else: msg = 'Timeout expecting trace file to be written "%s"' % fname raise IOError(msg) # # Ok... the file exists, but it might still be being written # req_res = open(fname, 'rb') request_dict, response_dict = msgpack.load(req_res) req_res.close() request = HTTPRequest.from_dict(request_dict) response = HTTPResponse.from_dict(response_dict) return (request, response)
def modify_request(self, request): ''' Mangles the request :param request: HTTPRequest instance that is going to be modified by the evasion plugin :return: The modified request ''' # First we mangle the URL qs = request.url_object.querystring.copy() qs = self._mutate(qs) # Finally, we set all the mutants to the request in order to return it new_url = request.url_object.copy() new_url.querystring = qs # Mangle the postdata post_data = request.get_data() if post_data: try: # Only mangle the postdata if it is a url encoded string post_data = parse_qs(post_data) except: pass else: post_data = str(self._mutate(post_data)) new_req = HTTPRequest(new_url, post_data, request.headers, request.get_origin_req_host()) return new_req
def test_modify_post_data(self): rc = rnd_case() u = URL('http://www.w3af.com/') r = HTTPRequest(u, data='a=b') modified_data = rc.modify_request(r).get_data() self.assertIn(modified_data, ['a=b', 'A=b', 'a=B', 'A=B'])
def test_encode_post_data(self): rhe = rnd_hex_encode() u = URL('http://www.w3af.com/') r = HTTPRequest(u, data='a=b') modified_pdata = rhe.modify_request(r).get_data() self.assertIn(modified_pdata, ['a=b', '%61=b', 'a=%62', '%61=%62'])
def test_add_when_qs(self): rp = rnd_param() u = URL('http://www.w3af.com/?id=1') r = HTTPRequest(u) qs = rp.modify_request(r).url_object.querystring self.assertEqual(len(qs), 2)
def test_path_file(self): rs = reversed_slashes() u = URL('http://www.w3af.com/abc/def.htm') r = HTTPRequest( u ) self.assertEqual(rs.modify_request( r ).url_object.url_string, u'http://www.w3af.com/abc\\def.htm')
def test_no_modification(self): rs = reversed_slashes() u = URL('http://www.w3af.com/') r = HTTPRequest( u ) self.assertEqual(rs.modify_request( r ).url_object.url_string, u'http://www.w3af.com/')
def __call__(self, uri, data=None, headers=Headers(), cache=False, grep=True, cookies=True): ''' :return: An HTTPResponse object that's the result of sending the request with a method different from "GET" or "POST". ''' if not isinstance(uri, URL): raise TypeError('The uri parameter of AnyMethod.' '__call__() must be of url.URL type.') if not isinstance(headers, Headers): raise TypeError('The headers parameter of AnyMethod.' '__call__() must be of Headers type.') self._xurllib._init() req = HTTPRequest(uri, data, cookies=cookies, cache=cache, method=self._method) req = self._xurllib._add_headers(req, headers or {}) return self._xurllib._send(req, grep=grep)
def test_encode_path_case01(self): rhe = rnd_hex_encode() u = URL('http://www.w3af.com/a/') r = HTTPRequest(u) modified_path = rhe.modify_request(r).url_object.get_path() self.assertIn(modified_path, ['/a/', '/%61/'])
def test_add_with_filename(self): rp = rnd_path() u = URL('http://www.w3af.com/abc/def.htm') r = HTTPRequest( u ) url_string = rp.modify_request( r ).url_object.url_string self.assertRegexpMatches(url_string, 'http://www.w3af.com/\w*/../abc/def.htm')
def test_add_when_dotdot(self): sosibd = shift_out_in_between_dots() u = URL('http://www.w3af.com/../') r = HTTPRequest( u ) self.assertEqual(sosibd.modify_request( r ).url_object.url_string, u'http://www.w3af.com/.%0E%0F./')
def test_no_modification(self): rhe = rnd_hex_encode() u = URL('http://www.w3af.com/') r = HTTPRequest(u) self.assertEqual( rhe.modify_request(r).url_object.url_string, u'http://www.w3af.com/')
def test_no_modification(self): modsec = mod_security() u = URL('http://www.w3af.com/') r = HTTPRequest(u) self.assertEqual( modsec.modify_request(r).url_object.url_string, u'http://www.w3af.com/')
def test_add_path_to_base_url(self): rp = rnd_path() u = URL('http://www.w3af.com/') r = HTTPRequest( u ) url_string = rp.modify_request( r ).url_object.url_string self.assertRegexpMatches(url_string, 'http://www.w3af.com/\w*/../')
def test_no_modification(self): sosibd = shift_out_in_between_dots() u = URL('http://www.w3af.com/') r = HTTPRequest( u ) self.assertEqual(sosibd.modify_request( r ).url_object.url_string, u'http://www.w3af.com/')
def test_no_modification(self): fwe = full_width_encode() u = URL('http://www.w3af.com/') r = HTTPRequest(u) self.assertEqual( fwe.modify_request(r).url_object.url_string, u'http://www.w3af.com/')
def test_modify_path(self): rc = rnd_case() u = URL('http://www.w3af.com/ab/') r = HTTPRequest(u) modified_path = rc.modify_request(r).url_object.get_path() self.assertIn(modified_path, ['/ab/', '/aB/', '/Ab/', '/AB/'])
def test_modify_basic(self): bbd = backspace_between_dots() u = URL('http://www.w3af.com/../') r = HTTPRequest(u) self.assertEqual( bbd.modify_request(r).url_object.url_string, u'http://www.w3af.com/.%41%08./')
def test_add_to_url_with_path(self): sr = self_reference() u = URL('http://www.w3af.com/abc/') r = HTTPRequest(u) self.assertEqual( sr.modify_request(r).url_object.url_string, u'http://www.w3af.com/./abc/./')
def http_request(self, req): url_instance = URL(req.get_full_url()) url_instance.set_param(self._url_parameter) new_request = HTTPRequest(url_instance, headers=req.headers, origin_req_host=req.get_origin_req_host(), unverifiable=req.is_unverifiable()) return new_request
def test_from_HTTPRequest_headers(self): hdr = Headers([('Foo', 'bar')]) request = HTTPRequest(self.url, headers=hdr) fr = create_fuzzable_request_from_request(request) self.assertEqual(fr.get_url(), self.url) self.assertEqual(fr.get_headers(), hdr) self.assertEqual(fr.get_method(), 'GET') self.assertIsInstance(fr, HTTPQSRequest)
def modifyRequest(self, request ): ''' Mangles the request @parameter request: HTTPRequest instance that is going to be modified by the evasion plugin @return: The modified request >>> from core.data.parsers.urlParser import url_object >>> modsec = modsecurity() >>> u = url_object('http://www.google.com/') >>> r = HTTPRequest( u ) >>> modsec.modifyRequest( r ).url_object.url_string 'http://www.google.com/' >>> u = url_object('http://www.google.com/') >>> r = HTTPRequest( u, data='' ) >>> modsec.modifyRequest( r ).get_data() '' >>> u = url_object('http://www.google.com/') >>> r = HTTPRequest( u, data='a=b' ) >>> modsec.modifyRequest( r ).get_data() '\\x00a=b' ''' # Mangle the postdata data = str(request.get_data()) if data: try: # Only mangle the postdata if it is a url encoded string parse_qs( data ) except: pass else: data = '\x00' + data headers_copy = copy.deepcopy(request.headers) headers_copy['content-length'] = str(len(data)) request = HTTPRequest( request.url_object, data, headers_copy, request.get_origin_req_host() ) return request
def test_handler_order_block(self): '''Get an instance of the extended urllib and verify that the blacklist handler still works, even when mixed with all the other handlers.''' # Configure the handler blocked_url = URL('http://moth/abc/def/') cf.cf.save('non_targets', [blocked_url,]) settings = opener_settings.OpenerSettings() settings.build_openers() opener = settings.get_custom_opener() request = HTTPRequest(blocked_url) request.url_object = blocked_url request.cookies = True request.get_from_cache = False response = opener.open(request) self.assertEqual(response.code, NO_CONTENT) self.assertEqual(response.id, 1)
def get_headers(self, uri): ''' :param uri: The URI we want to know the request headers :return: A Headers object with the HTTP headers that would be added by the library when sending a request to uri. ''' req = HTTPRequest(uri) req = self._add_headers(req) return Headers(req.headers)
def test_no_cache(self): url = URL('http://www.w3af.org') request = HTTPRequest(url, cache=False) cache = CacheHandler() self.assertEqual(cache.default_open(request), None) response = FakeHttplibHTTPResponse(200, 'OK', 'spameggs', Headers(), url.url_string) cache.http_response(request, response) self.assertEqual(cache.default_open(request), None)
def test_low_level(self): opener = urllib2.build_opener(CookieHandler) # With this request the CookieHandler should store a cookie in its # cookiejar set_cookie_req = HTTPRequest(self.URL_SENDS_COOKIE) opener.open(set_cookie_req).read() # And now it will send it because we're setting cookie to True with_cookie_req = HTTPRequest(self.URL_CHECK_COOKIE, cookies=True) with_cookie_res = opener.open(with_cookie_req).read() self.assertTrue('Cookie was sent.' in with_cookie_res) # And now it will NOT send any cookie because we're setting cookie to False no_cookie_req = HTTPRequest(self.URL_CHECK_COOKIE, cookies=False) no_cookie_res = opener.open(no_cookie_req).read() self.assertTrue('Cookie was NOT sent.' in no_cookie_res) # And now it will send it because we're setting cookie to True with_cookie_req = HTTPRequest(self.URL_CHECK_COOKIE, cookies=True) with_cookie_res = opener.open(with_cookie_req).read() self.assertTrue('Cookie was sent.' in with_cookie_res)
def setUp(self): create_temp_dir() self.plugin = sed() self.url = URL('http://www.w3af.com/') self.request = HTTPRequest(self.url)
class TestSed(unittest.TestCase): def setUp(self): create_temp_dir() self.plugin = sed() self.url = URL('http://www.w3af.com/') self.request = HTTPRequest(self.url) def tearDown(self): self.plugin.end() def test_blank_body(self): body = '' headers = Headers([('content-type', 'text/html')]) response = HTTPResponse(200, body, headers, self.url, self.url, _id=1) option_list = self.plugin.get_options() option_list['expressions'].set_value('qh/User/NotLuser/') self.plugin.set_options(option_list) mod_request = self.plugin.mangle_request(self.request) mod_response = self.plugin.mangle_response(response) self.assertEqual(mod_request.get_headers(), self.request.get_headers()) self.assertEqual(mod_response.get_headers(), response.get_headers()) self.assertEqual(mod_request.get_uri(), self.request.get_uri()) self.assertEqual(mod_response.get_uri(), response.get_uri()) self.assertEqual(mod_response.get_body(), response.get_body()) def test_response_body(self): body = 'hello user!' headers = Headers([('content-type', 'text/html')]) response = HTTPResponse(200, body, headers, self.url, self.url, _id=1) option_list = self.plugin.get_options() option_list['expressions'].set_value('sb/user/notluser/') self.plugin.set_options(option_list) mod_request = self.plugin.mangle_request(self.request) mod_response = self.plugin.mangle_response(response) self.assertEqual(mod_request.get_headers(), self.request.get_headers()) self.assertEqual(mod_response.get_headers(), response.get_headers()) self.assertEqual(mod_request.get_uri(), self.request.get_uri()) self.assertEqual(mod_response.get_uri(), response.get_uri()) self.assertEqual(mod_response.get_body(), 'hello notluser!') def test_request_headers(self): headers = Headers([('content-type', 'text/html')]) request = HTTPRequest(self.url, headers=headers) option_list = self.plugin.get_options() option_list['expressions'].set_value('qh/html/xml/') self.plugin.set_options(option_list) mod_request = self.plugin.mangle_request(request) value, _ = mod_request.get_headers().iget('content-type') self.assertEqual(value, 'text/xml')