def test_from_dict_encodings(self):
for body, charset in TEST_RESPONSES.values():
html = body.encode(charset)
resp = self.create_resp(Headers([('Content-Type', 'text/xml')]),
html)
msg = msgpack.dumps(resp.to_dict())
loaded_dict = msgpack.loads(msg)
loaded_resp = HTTPResponse.from_dict(loaded_dict)
self.assertEquals(
smart_unicode(html,
DEFAULT_CHARSET,
ESCAPED_CHAR,
on_error_guess=False), loaded_resp.body)
def _grep(self, request, response):
url_instance = request.url_object
domain = url_instance.get_domain()
if self._grep_queue_put is not None and\
domain in cf.cf.get('target_domains'):
# Create a fuzzable request based on the urllib2 request object
headers_inst = Headers(request.headers.items())
fr = create_fuzzable_request_from_parts(url_instance,
request.get_method(),
request.get_data(),
headers_inst)
self._grep_queue_put((fr, response))
def test_multipart_post(self):
boundary, post_data = multipart_encode([('a', 'bcd'), ], [])
headers = Headers([('content-length', str(len(post_data))),
('content-type', 'multipart/form-data; boundary=%s' % boundary)])
fr = create_fuzzable_request_from_parts(self.url, add_headers=headers,
post_data=post_data, method='POST')
self.assertEqual(fr.get_url(), self.url)
self.assertEqual(fr.get_headers(), headers)
self.assertTrue(
'multipart/form-data' in fr.get_headers()['content-type'])
self.assertEqual(fr.get_method(), 'POST')
self.assertEqual(fr.get_dc(), {'a': ['bcd', ]})
self.assertIsInstance(fr, HTTPPostDataRequest)
def test_ssn_with_complex_html(self):
'''
Test for false positive "...discloses a US Social Security Number: "12-56-1011"..."
'''
body = '''<select name="servers">
<option value="0" selected="selected">0</option>
<option value="1">1</option>
<option value="2-5">2-5</option>
<option value="6-10">6-10</option>
<option value="11-19">11-19</option>
<option value="20+">20+</option>
</select>'''
headers = Headers([('content-type', 'text/html')])
response = HTTPResponse(200, body, headers, self.url, self.url, _id=1)
self.plugin.grep(self.request, response)
self.assertEqual(len(kb.kb.get('ssn', 'ssn')), 0)
def test_find_vulns_case04(self):
'''
Test case in which we configure correctly policies for all directives.
'''
header_value = "default-src 'self';script-src 'self';object-src 'self';" \
"style-src 'self';img-src 'self';media-src 'self';" \
"frame-src 'self';font-src 'self';sandbox;" \
"form-action '/myCtx/act';connect-src 'self';"\
"plugin-types application/pdf;reflected-xss filter;"\
"script-nonce AABBCCDDEE;"
hrds = {CSP_HEADER_W3C: header_value}.items()
csp_headers = Headers(hrds)
http_response = HTTPResponse(200, '', csp_headers, self.url, self.url)
vulns = find_vulns(http_response)
self.assertEqual(len(vulns), 0)
def test_xmlrpc_post(self):
post_data = '''<methodCall>
<methodName>system.listMethods</methodName>
<params></params>
</methodCall>'''
headers = Headers([('content-length', str(len(post_data)))])
fr = create_fuzzable_request_from_parts(self.url, add_headers=headers,
post_data=post_data,
method='POST')
self.assertEqual(fr.get_url(), self.url)
self.assertEqual(fr.get_headers(), headers)
self.assertEqual(fr.get_method(), 'POST')
self.assertIsInstance(fr, XMLRPCRequest)
def test_unsafe_inline_enabled_no_case01(self):
'''
Test case in which site do not provides "unsafe-inline" related CSP
(no directive value "unsafe-inline").
'''
hrds = {}
hrds[CSP_HEADER_FIREFOX] = CSP_DIRECTIVE_SCRIPT + " 'self'"
hrds[CSP_HEADER_W3C_REPORT_ONLY] = CSP_DIRECTIVE_DEFAULT + \
" 'self';" + CSP_DIRECTIVE_REPORT_URI + " http://example.com"
hrds[CSP_HEADER_W3C] = CSP_DIRECTIVE_SCRIPT + " 'self';" + \
CSP_DIRECTIVE_REPORT_URI + " /myrelativeuri"
csp_headers = Headers(hrds.items())
http_response = HTTPResponse(200, '', csp_headers, self.url, self.url)
self.assertFalse(unsafe_inline_enabled(http_response))
def test_cache_control_http(self):
'''
No cache control, but the content is not sensitive (sent over http) so
no bug is stored in KB.
'''
body = 'abc'
url = URL('http://www.w3af.com/')
headers = Headers([('content-type', 'text/html')])
request = FuzzableRequest(url, method='GET')
resp = HTTPResponse(200, body, headers, url, url, _id=1)
self.plugin.grep(request, resp)
self.plugin.end()
infos = kb.kb.get('cache_control', 'cache_control')
self.assertEquals(len(infos), 0)
def test_no_code_disclosure_xml(self, *args):
body = '''
<?xml version="1.0"?>
<note>
<to>Tove</to>
<from>Jani</from>
<heading>Reminder</heading>
<body>Don't forget me this weekend!</body>
</note>'''
url = URL('http://www.w3af.com/')
headers = Headers([('content-type', 'text/html')])
response = HTTPResponse(200, body, headers, url, url, _id=1)
request = FuzzableRequest(url, method='GET')
self.plugin.grep(request, response)
self.assertEqual(
len(kb.kb.get('code_disclosure', 'code_disclosure')), 0)
def test_strange_headers_positive(self):
body = 'Hello world'
url = URL('http://www.w3af.com/')
headers = Headers([('content-type', 'text/html'),
('hello-world', 'yes!')])
request = FuzzableRequest(url, method='GET')
resp_positive = HTTPResponse(200, body, headers, url, url, _id=1)
self.plugin.grep(request, resp_positive)
infos = kb.kb.get('strange_headers', 'strange_headers')
self.assertEquals(len(infos), 1)
info = infos[0]
self.assertEqual(info.get_name(), 'Strange header')
self.assertEqual(info.get_url(), url)
def test_retrieve_csp_policies_with_special_policies_case01(self):
'''
Test case in which 2 policies are specified using special directives
with empty value.
'''
header_value = "sandbox ; script-nonce "
hrds = {CSP_HEADER_W3C: header_value}.items()
csp_headers = Headers(hrds)
http_response = HTTPResponse(200, '', csp_headers, self.url, self.url)
policies = retrieve_csp_policies(http_response)
self.assertEqual(len(policies), 2)
self.assertEqual(len(policies[CSP_DIRECTIVE_SANDBOX]), 1)
self.assertEqual(policies[CSP_DIRECTIVE_SANDBOX][0], "")
self.assertEqual(len(policies[CSP_DIRECTIVE_SCRIPT_NONCE]), 1)
self.assertEqual(policies[CSP_DIRECTIVE_SCRIPT_NONCE][0], "")
def build_cors_request(url, origin_header_value):
'''
Method to generate a "GET" CORS HTTP request based on input context.
:param url: a URL object object.
:param origin_header_value: Value of the "ORIGIN" HTTP request header
(if value is set to None then the "ORIGIN"
header is skipped).
:return: A fuzzable request that will be sent to @url and has
@origin_header_value in the Origin header.
'''
headers = Headers()
if origin_header_value is not None:
headers["Origin"] = origin_header_value.strip()
forged_req = FuzzableRequest(url, 'GET', headers=headers)
return forged_req
def set_headers(self, headers):
'''
Sets the headers and also analyzes them in order to get the response
mime type (text/html , application/pdf, etc).
:param headers: The headers dict.
'''
# Fix lowercase in header names from HTTPMessage
if isinstance(headers, httplib.HTTPMessage):
self._headers = Headers()
for header in headers.headers:
key, value = header.split(':', 1)
self._headers[key.strip()] = value.strip()
else:
self._headers = headers
# Set the type, for easy access.
self._doc_type = HTTPResponse.DOC_TYPE_OTHER
find_word = lambda w: content_type.find(w) != -1
content_type_hvalue, _ = self._headers.iget('content-type', None)
# we need exactly content type but not charset
if content_type_hvalue is not None:
try:
self._content_type = content_type_hvalue.split(';', 1)[0]
except:
msg = 'Invalid Content-Type value "%s" sent in HTTP response.'
om.out.debug(msg % (content_type_hvalue, ))
else:
content_type = self._content_type.lower()
# Set the doc_type
if content_type.count('image'):
self._doc_type = HTTPResponse.DOC_TYPE_IMAGE
elif content_type.count('pdf'):
self._doc_type = HTTPResponse.DOC_TYPE_PDF
elif content_type.count('x-shockwave-flash'):
self._doc_type = HTTPResponse.DOC_TYPE_SWF
elif any(
imap(find_word,
('text', 'html', 'xml', 'txt', 'javascript'))):
self._doc_type = HTTPResponse.DOC_TYPE_TEXT_OR_HTML
def new_no_content_resp(uri, add_id=False):
'''
Return a new NO_CONTENT HTTPResponse object.
:param uri: URI string or request object
'''
no_content_response = HTTPResponse(NO_CONTENT,
'',
Headers(),
uri,
uri,
msg='No Content')
if add_id:
no_content_response.id = consecutive_number_generator.inc()
return no_content_response
def test_dump_case03(self):
header_value = ''.join(chr(i) for i in xrange(256))
expected = u'\r\n'.join([
u'GET http://w3af.com/a/b/c.php HTTP/1.1',
u'Hola: %s' % smart_unicode(header_value), u'', u''
])
headers = Headers([(u'Hola', header_value)])
#TODO: Note that I'm passing a dc to the FuzzableRequest and it's not
# appearing in the dump. It might be a bug...
fr = FuzzableRequest(self.url,
method='GET',
dc={u'a': ['b']},
headers=headers)
self.assertEqual(fr.dump(), expected)
def test_clamav_empty(self, *args):
body = ''
url = URL('http://www.w3af.com/')
headers = Headers([('content-type', 'text/html')])
response = HTTPResponse(200, body, headers, url, url, _id=1)
request = FuzzableRequest(url, method='GET')
self.plugin.grep(request, response)
# Let the worker pool wait for the clamd response, this is done by
# the core when run in a real scan
self.plugin.worker_pool.close()
self.plugin.worker_pool.join()
findings = kb.kb.get('clamav', 'malware')
self.assertEqual(len(findings), 0, findings)
def test_analyze_cookies_with_httponly_case_sensitive_expires(self):
body = ''
url = URL('https://www.w3af.com/')
headers = {
'content-type':
'text/html',
'Set-Cookie':
'name2=value2; Expires=Wed, 09-Jun-2021 10:18:14 GMT;Secure;HttpOnly'
}
headers = Headers(headers.items())
response = HTTPResponse(200, body, headers, url, url, _id=1)
request = FuzzableRequest(url, method='GET')
self.plugin.grep(request, response)
self.assertEqual(len(kb.kb.get('analyze_cookies', 'cookies')), 1)
self.assertEqual(len(kb.kb.get('analyze_cookies', 'security')), 0)
def test_cache_control_correct_headers(self):
'''
Sensitive content with cache control headers so NO BUG is stored in KB.
'''
body = 'abc'
url = URL('https://www.w3af.com/')
headers = Headers([('content-type', 'text/html'),
('Pragma', 'No-cache'),
('Cache-Control', 'No-store')])
request = FuzzableRequest(url, method='GET')
resp = HTTPResponse(200, body, headers, url, url, _id=1)
self.plugin.grep(request, resp)
self.plugin.end()
infos = kb.kb.get('cache_control', 'cache_control')
self.assertEquals(len(infos), 0)
def test_object(self):
body = '''header
<OBJECT
classid="clsid:8AD9C840-044E-11D1-B3E9-00805F499D93"
width="200" height="200">
<PARAM name="code" value="Applet1.class">
</OBJECT>
footer'''
url = URL('http://www.w3af.com/')
headers = Headers([('content-type', 'text/html')])
response = HTTPResponse(200, body, headers, url, url, _id=1)
request = FuzzableRequest(url, method='GET')
self.plugin.grep(request, response)
self.assertEquals(len(kb.kb.get('objects', 'object')), 1)
i = kb.kb.get('objects', 'object')[0]
self.assertTrue('"object"' in i.get_desc())
def test_applet(self):
body = '''header
<APPLET code="XYZApp.class" codebase="html/" align="baseline"
width="200" height="200">
<PARAM name="model" value="models/HyaluronicAcid.xyz">
No Java 2 SDK, Standard Edition v 1.4.2 support for APPLET!!
</APPLET>
footer'''
url = URL('http://www.w3af.com/')
headers = Headers([('content-type', 'text/html')])
response = HTTPResponse(200, body, headers, url, url, _id=1)
request = FuzzableRequest(url, method='GET')
self.plugin.grep(request, response)
self.assertEquals(len(kb.kb.get('objects', 'applet')), 1)
i = kb.kb.get('objects', 'applet')[0]
self.assertTrue('"applet"' in i.get_desc())
def test_from_dict(self):
html = 'header <b>ABC</b>-<b>DEF</b>-<b>XYZ</b> footer'
headers = Headers([('Content-Type', 'text/html')])
orig_resp = self.create_resp(headers, html)
msg = msgpack.dumps(orig_resp.to_dict())
loaded_dict = msgpack.loads(msg)
loaded_resp = HTTPResponse.from_dict(loaded_dict)
self.assertEqual(orig_resp, loaded_resp)
orig_resp.__dict__.pop('_body_lock')
loaded_resp.__dict__.pop('_body_lock')
self.assertEqual(orig_resp.__dict__.values(),
loaded_resp.__dict__.values())
def GET(self,
uri,
data=None,
headers=Headers(),
cache=False,
grep=True,
cookies=True,
respect_size_limit=True):
'''
HTTP GET a URI using a proxy, user agent, and other settings
that where previously set in opener_settings.py .
:param uri: This is the URI to GET, with the query string included.
:param data: Only used if the uri parameter is really a URL. The data
will be converted into a string and set as the URL object
query string before sending.
:param headers: Any special headers that will be sent with this request
:param cache: Should the library search the local cache for a response
before sending it to the wire?
:param grep: Should grep plugins be applied to this request/response?
:param cookies: Send stored cookies in request (or not)
:return: An HTTPResponse object.
'''
if not isinstance(uri, URL):
raise TypeError(
'The uri parameter of ExtendedUrllib.GET() must be of '
'url.URL type.')
if not isinstance(headers, Headers):
raise TypeError(
'The header parameter of ExtendedUrllib.GET() must be of '
'Headers type.')
# Validate what I'm sending, init the library (if needed)
self._init()
if data:
uri = uri.copy()
uri.querystring = data
req = HTTPRequest(uri, cookies=cookies, cache=cache)
req = self._add_headers(req, headers)
with raise_size_limit(respect_size_limit):
return self._send(req, grep=grep)
def test_retrieve_csp_report_uri_yes(self):
'''
Test case in which site provides CSP report uri.
'''
hrds = {}
hrds[CSP_HEADER_FIREFOX] = CSP_DIRECTIVE_OBJECT + " 'self'"
hrds[CSP_HEADER_W3C_REPORT_ONLY] = CSP_DIRECTIVE_DEFAULT + \
" 'self';" + CSP_DIRECTIVE_REPORT_URI + " http://example.com"
hrds[CSP_HEADER_W3C] = CSP_DIRECTIVE_SCRIPT + " 'self';" + \
CSP_DIRECTIVE_REPORT_URI + " /myrelativeuri"
csp_headers = Headers(hrds.items())
http_response = HTTPResponse(200, '', csp_headers, self.url, self.url)
uri_set = retrieve_csp_report_uri(http_response)
self.assertEqual(len(uri_set), 2)
self.assertTrue("http://example.com" in uri_set)
self.assertTrue("/myrelativeuri" in uri_set)
def test_cache_control_no_headers(self):
'''
Sensitive content without cache control headers so bug is stored in KB.
'''
body = 'abc'
url = URL('https://www.w3af.com/')
headers = Headers([('content-type', 'text/html')])
request = FuzzableRequest(url, method='GET')
resp = HTTPResponse(200, body, headers, url, url, _id=1)
self.plugin.grep(request, resp)
self.plugin.end()
infos = kb.kb.get('cache_control', 'cache_control')
self.assertEquals(len(infos), 1)
info = infos[0]
self.assertEqual(info.get_name(), 'Missing cache control for HTTPS content')
def test_body_parse_form(self):
body = '''<form action="/foo.bar" method="POST">
A: <input name="a" />
B: <input name="b" value="123" />
</form>'''
headers = Headers([('content-type', 'text/html')])
http_response = HTTPResponse(200, body, headers, self.url, self.url)
post_request_lst = create_fuzzable_requests(http_response,
add_self=False)
self.assertEqual(len(post_request_lst), 1)
post_request = post_request_lst[0]
self.assertEqual(post_request.get_url().url_string,
'http://www.w3af.com/foo.bar')
self.assertEqual(post_request.get_data(), 'a=&b=123')
self.assertEqual(post_request.get_method(), 'POST')
self.assertFalse('content-type' in post_request.get_headers())
def test_image_with_image_content_type(self, *args):
'''
Verify that our plugins don't break when we send them an image.
'''
file_path = os.path.join('plugins', 'tests', 'grep', 'data',
'w3af.png')
body = file(file_path).read()
hdrs = Headers({'Content-Type': 'image/png'}.items())
response = HTTPResponse(200,
body,
hdrs,
self.url_inst,
self.url_inst,
_id=random.randint(1, 5000))
request = FuzzableRequest(self.url_inst)
for pinst in self._plugins:
pinst.grep(request, response)
def test_retrieve_csp_policies_with_special_policies_case02(self):
'''
Test case in which 2 policies are specified using special directives
with explicit values.
'''
header_value = "sandbox allow-forms allow-scripts ;"\
" script-nonce AABBCCDDEE"
hrds = {CSP_HEADER_W3C: header_value}.items()
csp_headers = Headers(hrds)
http_response = HTTPResponse(200, '', csp_headers, self.url, self.url)
policies = retrieve_csp_policies(http_response)
self.assertEqual(len(policies), 2)
self.assertEqual(len(policies[CSP_DIRECTIVE_SANDBOX]), 2)
self.assertTrue("allow-forms" in policies[CSP_DIRECTIVE_SANDBOX])
self.assertTrue("allow-scripts" in policies[CSP_DIRECTIVE_SANDBOX])
self.assertEqual(len(policies[CSP_DIRECTIVE_SCRIPT_NONCE]), 1)
self.assertEqual(policies[CSP_DIRECTIVE_SCRIPT_NONCE][0], "AABBCCDDEE")
def test_user_defined_regex(self):
body = '<html><head><script>xhr = new XMLHttpRequest(); xhr.open(GET, "data.txt", true);'
url = URL('http://www.w3af.com/')
headers = Headers([('content-type', 'text/html')])
response = HTTPResponse(200, body, headers, url, url, _id=1)
request = FuzzableRequest(url, method='GET')
options = self.plugin.get_options()
options['single_regex'].set_value('".*?"')
self.plugin.set_options(options)
self.plugin.grep(request, response)
self.assertEquals(
len(kb.kb.get('user_defined_regex', 'user_defined_regex')), 1)
info_obj = kb.kb.get('user_defined_regex', 'user_defined_regex')[0]
self.assertTrue(info_obj.get_desc().startswith(
'User defined regular expression "'))
self.assertIn('data.txt', info_obj.get_desc())
def test_response_body(self):
body = 'hello user!'
headers = Headers([('content-type', 'text/html')])
response = HTTPResponse(200, body, headers, self.url, self.url, _id=1)
option_list = self.plugin.get_options()
option_list['expressions'].set_value('sb/user/notluser/')
self.plugin.set_options(option_list)
mod_request = self.plugin.mangle_request(self.request)
mod_response = self.plugin.mangle_response(response)
self.assertEqual(mod_request.get_headers(), self.request.get_headers())
self.assertEqual(mod_response.get_headers(), response.get_headers())
self.assertEqual(mod_request.get_uri(), self.request.get_uri())
self.assertEqual(mod_response.get_uri(), response.get_uri())
self.assertEqual(mod_response.get_body(), 'hello notluser!')
def test_no_code_disclosure(self, *args):
body = """Lorem ipsum dolor sit amet, consectetuer adipiscing elit. Integer
eu lacus accumsan arcu fermentum euismod. Donec pulvinar porttitor
tellus. Aliquam venenatis. Donec facilisis pharetra tortor. In nec
mauris eget magna consequat convallis. Nam sed sem vitae odio
pellentesque interdum. Sed consequat viverra nisl. Suspendisse arcu
metus, blandit quis, rhoncus <a>,</a> pharetra eget, velit. Mauris
urna. Morbi nonummy molestie orci. Praesent nisi elit, fringilla ac,
suscipit non, tristique vel, ma<?uris. Curabitur vel lorem id nisl porta
adipiscing. Suspendisse eu lectus. In nunc. Duis vulputate tristique
enim. Donec quis lectus a justo imperdiet tempus."""
url = URL('http://www.w3af.com/')
headers = Headers([('content-type', 'text/html')])
response = HTTPResponse(200, body, headers, url, url, _id=1)
request = FuzzableRequest(url, method='GET')
self.plugin.grep(request, response)
self.assertEqual(
len(kb.kb.get('code_disclosure', 'code_disclosure')), 0)
