def test_simplest(self): fr = FuzzableRequest.from_parts(self.url) self.assertEqual(fr.get_url(), self.url) self.assertEqual(fr.get_headers(), Headers()) self.assertEqual(fr.get_method(), 'GET') self.assertIsInstance(fr.get_raw_data(), KeyValueContainer)
def test_str_with_postdata(self): headers = Headers([('content-type', URLEncodedForm.ENCODING)]) fr = FuzzableRequest.from_parts("http://www.w3af.com/", post_data='a=1', headers=headers) expected = 'Method: GET | http://www.w3af.com/ | URL encoded ' \ 'form: (a)' self.assertEqual(str(fr), expected)
def test_from_freq(self): freq = FuzzableRequest.from_parts(self.get_url(), 'POST', XML_WITH_FUZZABLE, Headers()) m = XmlRpcMutant(freq) self.assertIsInstance(m.get_dc(), XmlRpcContainer) dc = m.get_dc() self.assertIn('string', dc) self.assertIn('base64', dc) self.assertEqual(len(dc['string']), 1) self.assertEqual(len(dc['base64']), 1) self.assertEqual(dc['string'][0], 'Foo bar') self.assertEqual(dc['base64'][0], 'Spam eggs') self.assertEqual(str(m.get_dc()), str(m.get_data())) found_at = '"http://w3af.com/a/b/c.php", using HTTP method POST. ' \ 'The sent XML-RPC was: "<methodCall>\n <methodName>' \ 'sample.sum</methodName>\n <params>\n ".' self.assertEqual(m.found_at(), found_at) headers = m.get_headers() self.assertIn('Content-Type', headers) self.assertEqual(headers['Content-Type'], 'application/xml')
def test_multipart_post(self): boundary, post_data = multipart_encode([ ('a', 'bcd'), ], []) multipart_boundary = 'multipart/form-data; boundary=%s' headers = Headers([('content-length', str(len(post_data))), ('content-type', multipart_boundary % boundary)]) fr = FuzzableRequest.from_parts(self.url, headers=headers, post_data=post_data, method='POST') form_params = FormParameters() form_params.add_field_by_attr_items([('name', 'a'), ('type', 'text'), ('value', 'bcd')]) expected_container = MultipartContainer(form_params) expected_headers = Headers([('content-type', multipart_boundary % boundary)]) self.assertEqual(fr.get_url(), self.url) self.assertEqual(fr.get_headers(), expected_headers) self.assertIn('multipart/form-data', fr.get_headers()['content-type']) self.assertEqual(fr.get_method(), 'POST') self.assertIsInstance(fr.get_raw_data(), MultipartContainer) self.assertEqual(fr.get_raw_data(), expected_container)
def _get_request_response_from_work_unit(self, work_unit): """ In some cases the work unit is a tuple with request / response instances. In other cases it is an ID, which needs to be queried from the History DB to get the request / response. :param work_unit: One of the options explained above :return: A request / response tuple """ if not isinstance(work_unit, int): request, response = work_unit else: # Before we sent requests and responses as work units, # but since we changed from Queue to CachedQueue for BaseConsumer # the database was growing really big (1GB) for storing that traffic # and I decided to migrate to using just the response.id and querying # the SQLite one extra time. history = HistoryItem() request, response = history.load_from_file(work_unit) # Create a fuzzable request based on the urllib2 request object headers_inst = Headers(request.header_items()) request = FuzzableRequest.from_parts(request.url_object, request.get_method(), request.get_data() or '', headers_inst) return request, response
def test_simplest(self): fr = FuzzableRequest.from_parts(self.url) self.assertEqual(fr.get_url(), self.url) self.assertEqual(fr.get_headers(), Headers()) self.assertEqual(fr.get_method(), "GET") self.assertIsInstance(fr.get_raw_data(), KeyValueContainer)
def test_multipart_post(self): boundary, post_data = multipart_encode([('a', 'bcd'), ], []) multipart_boundary = 'multipart/form-data; boundary=%s' headers = Headers([('content-length', str(len(post_data))), ('content-type', multipart_boundary % boundary)]) fr = FuzzableRequest.from_parts(self.url, headers=headers, post_data=post_data, method='POST') form_params = FormParameters() form_params.add_field_by_attr_items([('name', 'a'), ('type', 'text'), ('value', 'bcd')]) expected_container = MultipartContainer(form_params) expected_headers = Headers([('content-type', multipart_boundary % boundary)]) self.assertEqual(fr.get_url(), self.url) self.assertEqual(fr.get_headers(), expected_headers) self.assertIn('multipart/form-data', fr.get_headers()['content-type']) self.assertEqual(fr.get_method(), 'POST') self.assertIsInstance(fr.get_raw_data(), MultipartContainer) self.assertEqual(fr.get_raw_data(), expected_container)
def test_headers_method(self): hdr = Headers([('foo', 'bar')]) fr = FuzzableRequest.from_parts(self.url, method='PUT', headers=hdr) self.assertEqual(fr.get_url(), self.url) self.assertEqual(fr.get_headers(), hdr) self.assertEqual(fr.get_method(), 'PUT') self.assertIsInstance(fr.get_raw_data(), KeyValueContainer)
def test_headers_method(self): hdr = Headers([("foo", "bar")]) fr = FuzzableRequest.from_parts(self.url, method="PUT", headers=hdr) self.assertEqual(fr.get_url(), self.url) self.assertEqual(fr.get_headers(), hdr) self.assertEqual(fr.get_method(), "PUT") self.assertIsInstance(fr.get_raw_data(), KeyValueContainer)
def test_raw_url(self): raw_url = "http://w3af.org/foo/" fr = FuzzableRequest.from_parts(raw_url) self.assertEqual(fr.get_url().url_string, raw_url) self.assertEqual(fr.get_headers(), Headers()) self.assertEqual(fr.get_method(), "GET") self.assertIsInstance(fr.get_raw_data(), KeyValueContainer)
def test_raw_url(self): raw_url = 'http://w3af.org/foo/' fr = FuzzableRequest.from_parts(raw_url) self.assertEqual(fr.get_url().url_string, raw_url) self.assertEqual(fr.get_headers(), Headers()) self.assertEqual(fr.get_method(), 'GET') self.assertIsInstance(fr.get_raw_data(), KeyValueContainer)
def test_json_post(self): post_data = '{"1":"2"}' hdr = Headers([("content-length", str(len(post_data))), ("content-type", "application/json")]) fr = FuzzableRequest.from_parts(self.url, headers=hdr, post_data=post_data, method="POST") self.assertEqual(fr.get_url(), self.url) self.assertEqual(fr.get_headers(), hdr) self.assertEqual(fr.get_method(), "POST") self.assertIsInstance(fr.get_raw_data(), JSONContainer)
def test_xmlrpc_mutant(self): url = URL('http://moth/?id=1') post_data = XML_WITH_FUZZABLE headers = Headers() freq = FuzzableRequest.from_parts(url, 'POST', post_data, headers) mutants = create_mutants(freq, self.payloads) self.assertAllInstance(mutants[:2], QSMutant) self.assertAllInstance(mutants[4:], XmlRpcMutant) self.assertAllHaveTokens(mutants)
class import_sqlite(CrawlPlugin): """ Import requests (method,uri,headers,data) stored in sqlite database. :author: @s0i37 """ def __init__(self): CrawlPlugin.__init__(self) self._input_sqlite = '' @runonce(exc_class=RunOnce) def crawl(self, fuzzable_request): """ Read the input file, and create the fuzzable_request_list based on that information. :param fuzzable_request: A fuzzable_request instance that contains (among other things) the URL to test. In this case it is simply ignored and data is read from the input files. """ self._load_data_from_sqlite() def _load_data_from_sqlite(self): """ Load data from the csv file """ if not self._input_sqlite: return try: db = sqlite3.connect(self._input_sqlite) except BaseFrameworkException, e: msg = 'An error was found while trying to read "%s": "%s".' om.out.error(msg % (self._input_csv, e)) return sql = db.cursor() for method, uri, headers, data in sql.execute( "select method,uri,headers,data from requests"): try: self.debug("+ %s %s" % (method, uri)) headers = Headers.from_string(str(headers)) self.output_queue.put( FuzzableRequest.from_parts(uri, method=method, post_data=str(data), headers=headers)) except Exception as e: import traceback traceback.print_exc() msg = 'import_sqlite: %s' self.debug(msg % str(e)) db.close()
def test_simple_post(self): post_data = "a=b&d=3" hdr = Headers([("content-length", str(len(post_data))), ("content-type", URLEncodedForm.ENCODING)]) fr = FuzzableRequest.from_parts(self.url, headers=hdr, post_data=post_data, method="POST") self.assertEqual(fr.get_url(), self.url) self.assertEqual(fr.get_headers(), hdr) self.assertEqual(fr.get_method(), "POST") self.assertIn("content-type", fr.get_headers()) self.assertIsInstance(fr.get_raw_data(), URLEncodedForm)
def test_json_post(self): post_data = '{"1":"2"}' hdr = Headers([('content-length', str(len(post_data))), ('content-type', 'application/json')]) fr = FuzzableRequest.from_parts(self.url, headers=hdr, post_data=post_data, method='POST') self.assertEqual(fr.get_url(), self.url) self.assertEqual(fr.get_headers(), hdr) self.assertEqual(fr.get_method(), 'POST') self.assertIsInstance(fr.get_raw_data(), JSONContainer)
def test_simple_post(self): post_data = 'a=b&d=3' hdr = Headers([('content-length', str(len(post_data))), ('content-type', URLEncodedForm.ENCODING)]) fr = FuzzableRequest.from_parts(self.url, headers=hdr, post_data=post_data, method='POST') self.assertEqual(fr.get_url(), self.url) self.assertEqual(fr.get_headers(), hdr) self.assertEqual(fr.get_method(), 'POST') self.assertIn('content-type', fr.get_headers()) self.assertIsInstance(fr.get_raw_data(), URLEncodedForm)
def test_xmlrpc_post(self): post_data = """<methodCall> <methodName>system.listMethods</methodName> <params></params> </methodCall>""" headers = Headers([("content-length", str(len(post_data)))]) fr = FuzzableRequest.from_parts(self.url, headers=headers, post_data=post_data, method="POST") self.assertEqual(fr.get_url(), self.url) self.assertEqual(fr.get_headers(), headers) self.assertEqual(fr.get_method(), "POST") self.assertIsInstance(fr.get_raw_data(), XmlRpcContainer)
def _grep(self, request, response): url_instance = request.url_object domain = url_instance.get_domain() if self._grep_queue_put is not None and \ domain in cf.cf.get('target_domains'): # Create a fuzzable request based on the urllib2 request object headers_inst = Headers(request.header_items()) fr = FuzzableRequest.from_parts(url_instance, request.get_method(), request.get_data() or '', headers_inst) self._grep_queue_put((fr, response))
def create_mutant_from_params(method, uri, var, post_data): uri = URL(uri) if method.upper() == 'GET' and var in uri.querystring: MutantKlass = QSMutant headers = Headers() else: MutantKlass = PostDataMutant headers = Headers([('content-type', URLEncodedForm.ENCODING)]) freq = FuzzableRequest.from_parts(uri, method=method, post_data=post_data, headers=headers) mutant = MutantKlass(freq) mutant.get_dc().set_token((var, 0)) return mutant
def test_xmlrpc_post(self): post_data = """<methodCall> <methodName>system.listMethods</methodName> <params></params> </methodCall>""" headers = Headers([('content-length', str(len(post_data)))]) fr = FuzzableRequest.from_parts(self.url, headers=headers, post_data=post_data, method='POST') self.assertEqual(fr.get_url(), self.url) self.assertEqual(fr.get_headers(), headers) self.assertEqual(fr.get_method(), 'POST') self.assertIsInstance(fr.get_raw_data(), XmlRpcContainer)
def test_json_creation_missing_header(self): post_data = '{"1":"2"}' # Missing the content-type header for json headers = Headers([("content-length", str(len(post_data)))]) fr = FuzzableRequest.from_parts(self.url, headers=headers, post_data=post_data, method="POST") self.assertEqual(fr.get_url(), self.url) self.assertEqual(fr.get_headers(), headers) self.assertEqual(fr.get_method(), "POST") # Here the "default" post-data is set, which will be empty because we # failed to parse the post-data self.assertIsInstance(fr.get_raw_data(), PlainContainer) self.assertEqual(fr.get_raw_data().get_param_names(), [])
def _grep(self, request, response): url_instance = request.url_object domain = url_instance.get_domain() if self._grep_queue_put is not None and\ domain in cf.cf.get('target_domains'): # Create a fuzzable request based on the urllib2 request object headers_inst = Headers(request.header_items()) fr = FuzzableRequest.from_parts(url_instance, request.get_method(), request.get_data() or '', headers_inst) self._grep_queue_put((fr, response))
def test_json_creation_missing_header(self): post_data = '{"1":"2"}' # Missing the content-type header for json headers = Headers([('content-length', str(len(post_data)))]) fr = FuzzableRequest.from_parts(self.url, headers=headers, post_data=post_data, method='POST') self.assertEqual(fr.get_url(), self.url) self.assertEqual(fr.get_headers(), headers) self.assertEqual(fr.get_method(), 'POST') # Here the "default" post-data is set, which will be empty because we # failed to parse the post-data self.assertIsInstance(fr.get_raw_data(), PlainContainer) self.assertEqual(fr.get_raw_data().get_param_names(), [])
def create_fuzzable_request_from_request(request, add_headers=None): """ :return: A fuzzable request with the same info as request """ if not isinstance(request, HTTPRequest): raise TypeError('Requires HTTPRequest to create FuzzableRequest.') url = request.url_object post_data = str(request.get_data() or '') method = request.get_method() headers = Headers(request.headers.items()) headers.update(request.unredirected_hdrs.items()) headers.update(add_headers or Headers()) return FuzzableRequest.from_parts(url, method=method, post_data=post_data, headers=headers)
def test_invalid_multipart_post(self): _, post_data = multipart_encode([("a", "bcd")], []) # It is invalid because there is a missing boundary parameter in the # content-type header headers = Headers([("content-length", str(len(post_data))), ("content-type", "multipart/form-data")]) fr = FuzzableRequest.from_parts(self.url, headers=headers, post_data=post_data, method="POST") self.assertEqual(fr.get_url(), self.url) self.assertEqual(fr.get_headers(), headers) self.assertEqual(fr.get_method(), "POST") # Here the "default" post-data is set, which will be empty because we # failed to parse the post-data self.assertIsInstance(fr.get_raw_data(), PlainContainer) self.assertEqual(fr.get_raw_data().get_param_names(), [])
def test_invalid_multipart_post(self): _, post_data = multipart_encode([('a', 'bcd'), ], []) # It is invalid because there is a missing boundary parameter in the # content-type header headers = Headers([('content-length', str(len(post_data))), ('content-type', 'multipart/form-data')]) fr = FuzzableRequest.from_parts(self.url, headers=headers, post_data=post_data, method='POST') self.assertEqual(fr.get_url(), self.url) self.assertEqual(fr.get_headers(), headers) self.assertEqual(fr.get_method(), 'POST') # Here the "default" post-data is set, which will be empty because we # failed to parse the post-data self.assertIsInstance(fr.get_raw_data(), PlainContainer) self.assertEqual(fr.get_raw_data().get_param_names(), [])
def _get_request_response_from_id_impl(self, http_response_id): """ Just reads the request and response from the files. No threading, events, caching, etc. :param http_response_id: The HTTP response ID :return: An HTTP request and response tuple """ history = HistoryItem() request, response = history.load_from_file(http_response_id) # Create a fuzzable request based on the urllib2 request object headers_inst = Headers(request.header_items()) request = FuzzableRequest.from_parts(request.url_object, request.get_method(), request.get_data() or '', headers_inst) return request, response
def test_multipart_post(self): boundary, post_data = multipart_encode([("a", "bcd")], []) multipart_boundary = "multipart/form-data; boundary=%s" headers = Headers([("content-length", str(len(post_data))), ("content-type", multipart_boundary % boundary)]) fr = FuzzableRequest.from_parts(self.url, headers=headers, post_data=post_data, method="POST") form_params = FormParameters() form_params.add_input([("name", "a"), ("type", "text"), ("value", "bcd")]) expected_container = MultipartContainer(form_params) expected_headers = Headers([("content-type", multipart_boundary % boundary)]) self.assertEqual(fr.get_url(), self.url) self.assertEqual(fr.get_headers(), expected_headers) self.assertIn("multipart/form-data", fr.get_headers()["content-type"]) self.assertEqual(fr.get_method(), "POST") self.assertIsInstance(fr.get_raw_data(), MultipartContainer) self.assertEqual(fr.get_raw_data(), expected_container)
def mangle_request(self, request): """ This method mangles the request. :param request: This is the request to mangle. :return: A mangled version of the request. """ data = request.get_data() for regex, string in self._manglers['q']['b']: data = regex.sub(string, data) header_string = str(request.get_headers()) for regex, string in self._manglers['q']['h']: header_string = regex.sub(string, header_string) headers_inst = Headers.from_string(header_string) return FuzzableRequest.from_parts(request.get_uri(), method=request.get_method(), post_data=data, headers=headers_inst)
def _create_fuzzable_request(self): """ Based on the attributes, return a fuzzable request object. Important variables used here: - self.headers : Stores the headers for the request - self.rfile : A file like object that stores the post_data - self.path : Stores the URL that was requested by the browser """ # See HTTPWrapperClass if hasattr(self.server, 'chainedHandler'): base_path = "https://" + self.server.chainedHandler.path path = base_path + self.path else: path = self.path headers = Headers(self.headers.dict.items()) post_data = self._get_post_data() return FuzzableRequest.from_parts(path, method=self.command, post_data=post_data, headers=headers)
def test_multipart_fuzzable_request_store(self): boundary, post_data = multipart_encode([('a', 'bcd'), ], []) multipart_boundary = MultipartContainer.MULTIPART_HEADER headers = Headers([('content-length', str(len(post_data))), ('content-type', multipart_boundary % boundary)]) dc = MultipartContainer.from_postdata(headers, post_data) post_data = str(dc) fr = FuzzableRequest.from_parts(URL('http://www.w3af.com/'), method='POST', post_data=post_data, headers=headers) disk_set = DiskSet() disk_set.add(fr) fr_read = disk_set[0] self.assertIsInstance(fr_read.get_raw_data(), MultipartContainer) self.assertIn('a', fr_read.get_raw_data())
msg = 'The file format is incorrect, an error was found while'\ ' parsing: "%s". Exception: "%s".' om.out.error(msg % (csv_row, value_error)) else: # Create the obj based on the information uri = URL(uri) if not uri.is_valid_domain(): return # If there is postdata, force parsing using urlencoded form headers = None if postdata: headers = Headers([('content-type', URLEncodedForm.ENCODING)]) return FuzzableRequest.from_parts(uri, method=method, post_data=postdata, headers=headers) def _objs_from_burp_log(self, burp_file): """ Read a burp log (XML) and extract the information. """ xp = BurpParser() parser = etree.XMLParser(target=xp) try: requests = etree.fromstring(file(burp_file).read(), parser) except XMLSyntaxError, xse: msg = 'The Burp input file is not a valid XML document. The' \ ' parser error is: "%s"' om.out.error(msg % xse)
version = first_line[-1] uri = ' '.join(first_line[1:-1]) check_version_syntax(version) # If we got here, we have a nice method, uri, version first line # Now we parse the headers (easy!) and finally we send the request headers_str = split_head[1:] headers_inst = Headers() for header in headers_str: one_split_header = header.split(':', 1) if len(one_split_header) == 1: msg = 'The HTTP request has an invalid header: "%s".' raise BaseFrameworkException(msg % header) header_name = one_split_header[0].strip() header_value = one_split_header[1].strip() if header_name in headers_inst: headers_inst[header_name] += ', ' + header_value else: headers_inst[header_name] = header_value host, _ = headers_inst.iget('host', None) try: uri = URL(check_uri_syntax(uri, host)) except ValueError, ve: raise BaseFrameworkException(str(ve)) return FuzzableRequest.from_parts(uri, method, postdata, headers_inst)
version = first_line[-1] uri = " ".join(first_line[1:-1]) check_version_syntax(version) # If we got here, we have a nice method, uri, version first line # Now we parse the headers (easy!) and finally we send the request headers_str = split_head[1:] headers_inst = Headers() for header in headers_str: one_split_header = header.split(":", 1) if len(one_split_header) == 1: msg = 'The HTTP request has an invalid header: "%s".' raise BaseFrameworkException(msg % header) header_name = one_split_header[0].strip() header_value = one_split_header[1].strip() if header_name in headers_inst: headers_inst[header_name] += ", " + header_value else: headers_inst[header_name] = header_value host, _ = headers_inst.iget("host", None) try: uri = URL(check_uri_syntax(uri, host)) except ValueError, ve: raise BaseFrameworkException(str(ve)) return FuzzableRequest.from_parts(uri, method, postdata, headers_inst)