def test_get_short_printable_repr_unicode_value_unicode(self):
dc = KeyValueContainer([(u'aéb', [u'céd']), (u'b', ['2', '3'])])
dc.MAX_PRINTABLE = 7
token = DataToken(u'aéb', u'céd', (u'aéb', 0))
dc.set_token(token)
self.assertEqual(dc.get_short_printable_repr(), '...a.b=c.d...')
def test_iter_bound_tokens(self):
dc = KeyValueContainer([(u'a', ['1']), (u'b', ['2', '3'])])
dcc_tokens = [(dcc, t) for dcc, t in dc.iter_bound_tokens()]
EXPECTED_TOKENS = [('a', '1'), ('b', '2'), ('b', '3')]
token_data = [(t.get_name(), t.get_value()) for dcc, t in dcc_tokens]
self.assertEqual(EXPECTED_TOKENS, token_data)
for dcc, _ in dcc_tokens:
self.assertIsInstance(dcc, KeyValueContainer)
self.assertEquals(dcc, dc)
self.assertEqual(str(dcc), 'a=1&b=2&b=3')
only_dcc = [dcc for dcc, t in dcc_tokens]
dcc = only_dcc[0]
token = dcc.get_token()
token.set_value('5')
self.assertEqual(str(dcc), 'a=5&b=2&b=3')
dcc = only_dcc[1]
token = dcc.get_token()
token.set_value('5')
self.assertEqual(str(dcc), 'a=1&b=5&b=3')
dcc = only_dcc[2]
token = dcc.get_token()
token.set_value('5')
self.assertEqual(str(dcc), 'a=1&b=2&b=5')
def __init__(self,
uri,
method='GET',
headers=None,
cookie=None,
post_data=None):
super(FuzzableRequest, self).__init__()
# Note: Do not check for the URI/Headers type here, since I'm doing it
# in set_uri() and set_headers() already.
if cookie is not None and not isinstance(cookie, Cookie):
raise TypeError(TYPE_ERROR % ('cookie', 'Cookie'))
if post_data is not None and not isinstance(post_data, DataContainer):
raise TypeError(TYPE_ERROR % ('post_data', 'DataContainer'))
# Internal variables
self._method = method
self._cookie = Cookie() if cookie is None else cookie
self._post_data = KeyValueContainer(
) if post_data is None else post_data
# Set the headers
self._headers = None
pheaders = Headers() if headers is None else headers
self.set_headers(pheaders)
# Set the URL
self._uri = None
self._url = None
self.set_uri(uri)
# Set the internal variables
self._sent_info_comp = None
def test_get_short_printable_repr_unicode_value(self):
dc = KeyValueContainer([(u'a', ['x']), (u'b', ['2', '3'])])
dc.MAX_PRINTABLE = 5
token = DataToken('a', 'é', ('a', 0))
dc.set_token(token)
self.assertEqual(dc.get_short_printable_repr(), '...a=....')
def test_get_short_printable_repr_token_obj(self):
dc = KeyValueContainer([(u'a', ['1']), (u'b', ['2', '3'])])
token = DataToken('a', '1', ('a', 0))
dc.set_token(token)
self.assertIsNotNone(dc.get_token())
self.assertEqual(dc.get_short_printable_repr(), 'a=1&b=2&b=3')
def test_double_data_token_wrap(self):
dc = KeyValueContainer([(u'a', ['1']), (u'b', ['c', '3'])])
dc.set_token(('b', 1))
for dcc, token in dc.iter_bound_tokens():
self.assertIsInstance(token, DataToken)
self.assertIsInstance(token.get_value(), basestring)
def test_iter_tokens(self):
dc = KeyValueContainer([(u'a', ['1']), (u'b', ['2', '3'])])
tokens = [t for t in dc.iter_tokens()]
EXPECTED_TOKENS = [('a', '1'), ('b', '2'), ('b', '3')]
token_data = [(t.get_name(), t.get_value()) for t in tokens]
self.assertEqual(EXPECTED_TOKENS, token_data)
def test_iter_bound_tokens(self):
dc = KeyValueContainer([(u'a', ['1']), (u'b', ['2', '3'])])
dcc_tokens = [(dcc, t) for dcc, t in dc.iter_bound_tokens()]
EXPECTED_TOKENS = [('a', '1'), ('b', '2'), ('b', '3')]
token_data = [(t.get_name(), t.get_value()) for dcc, t in dcc_tokens]
self.assertEqual(EXPECTED_TOKENS, token_data)
for dcc, _ in dcc_tokens:
self.assertIsInstance(dcc, KeyValueContainer)
self.assertEquals(dcc, dc)
self.assertEqual(str(dcc), 'a=1&b=2&b=3')
only_dcc = [dcc for dcc, t in dcc_tokens]
dcc = only_dcc[0]
token = dcc.get_token()
token.set_value('5')
self.assertEqual(str(dcc), 'a=5&b=2&b=3')
dcc = only_dcc[1]
token = dcc.get_token()
token.set_value('5')
self.assertEqual(str(dcc), 'a=1&b=5&b=3')
dcc = only_dcc[2]
token = dcc.get_token()
token.set_value('5')
self.assertEqual(str(dcc), 'a=1&b=2&b=5')
def test_double_data_token_wrap(self):
dc = KeyValueContainer([(u'a', ['1']), (u'b', ['c', '3'])])
dc.set_token(('b', 1))
for dcc, token in dc.iter_bound_tokens():
self.assertIsInstance(token, DataToken)
self.assertIsInstance(token.get_value(), basestring)
def test_iter_tokens(self):
dc = KeyValueContainer([(u'a', ['1']), (u'b', ['2', '3'])])
tokens = [t for t in dc.iter_tokens()]
EXPECTED_TOKENS = [('a', '1'), ('b', '2'), ('b', '3')]
token_data = [(t.get_name(), t.get_value()) for t in tokens]
self.assertEqual(EXPECTED_TOKENS, token_data)
def test_copy_with_token(self):
dc = KeyValueContainer([(u'a', ['1']), (u'b', ['2', '3'])])
dc.set_token(('a', 0))
dc_copy = copy.deepcopy(dc)
self.assertEqual(dc.get_token(), dc_copy.get_token())
self.assertIsNotNone(dc.get_token())
self.assertIsNotNone(dc_copy.get_token())
def test_str(self):
dc = KeyValueContainer([(u'a', ['1']), (u'b', ['2', '3'])])
str_dc = str(dc)
self.assertEqual(str_dc, 'a=1&b=2&b=3')
self.assertIsInstance(str_dc, str)
dc = KeyValueContainer([(u'aaa', [''])])
self.assertEqual(str(dc), 'aaa=')
dc = KeyValueContainer([(u'aaa', ('', ''))])
self.assertEqual(str(dc), 'aaa=&aaa=')
def __init__(self, xml_post_data, encoding=UTF8):
"""
:param xml_post_data: The XMLRPC call as string
"""
KeyValueContainer.__init__(self, init_val=[], encoding=encoding)
if not isinstance(xml_post_data, basestring):
raise TypeError(ERR_MSG % xml_post_data)
self._xml_post_data = xml_post_data
self.parse_xml_data(xml_post_data)
def __init__(self, xml_post_data, encoding=UTF8):
"""
:param xml_post_data: The XMLRPC call as string
"""
KeyValueContainer.__init__(self, init_val=[], encoding=encoding)
if not isinstance(xml_post_data, basestring):
raise TypeError(ERR_MSG % xml_post_data)
self._xml_post_data = xml_post_data
self.parse_xml_data(xml_post_data)
def __init__(self, uri, method='GET', headers=None, cookie=None,
post_data=None):
super(FuzzableRequest, self).__init__()
# Note: Do not check for the URI/Headers type here, since I'm doing it
# in set_uri() and set_headers() already.
if cookie is not None and not isinstance(cookie, Cookie):
raise TypeError(TYPE_ERROR % ('cookie', 'Cookie'))
if post_data is not None and not isinstance(post_data, DataContainer):
raise TypeError(TYPE_ERROR % ('post_data', 'DataContainer'))
# Internal variables
self._method = method
self._cookie = Cookie() if cookie is None else cookie
self._post_data = KeyValueContainer() if post_data is None else post_data
# Set the headers
self._headers = None
pheaders = Headers() if headers is None else headers
self.set_headers(pheaders)
# Set the URL
self._uri = None
self._url = None
self.set_uri(uri)
# Set the internal variables
self._sent_info_comp = None
def test_clean_form_fuzzable_request(self):
fr = FuzzableRequest(URL("http://www.w3af.com/"),
headers=Headers([('Host', 'www.w3af.com')]),
method='POST',
post_data=KeyValueContainer(init_val=[('data', ['23'])]))
expected = u'(POST)-http://www.w3af.com/!data=number'
self.assertEqual(clean_fuzzable_request(fr), expected)
def test_basic(self):
dc = KeyValueContainer([(u'a', ['1']), (u'b', ['2', '3'])])
self.assertIn('a', dc)
self.assertIn('b', dc)
self.assertEqual(dc['a'], ['1'])
self.assertEqual(dc['b'], ['2', '3'])
def get_fuzzable_request(self):
headers = Headers([(u'Hello', u'World')])
post_data = KeyValueContainer(init_val=[('a', ['b'])])
url = URL('http://w3af.org')
return FuzzableRequest(url,
method='GET',
post_data=post_data,
headers=headers)
def test_export_import_with_post_data(self):
dc = KeyValueContainer(init_val=[('a', ['1'])])
fr = FuzzableRequest(URL("http://www.w3af.com/"), post_data=dc)
self.assertEqual(fr.to_csv(), '"GET","http://www.w3af.com/","a=1"')
raise SkipTest('Failing because we do NOT export headers')
imported_fr = fr.from_csv(fr.to_csv())
self.assertEqual(imported_fr, fr)
def test_get_short_printable_repr_token_obj_reduce_printable(self):
dc = KeyValueContainer([(u'a', ['1']), (u'b', ['2', '3'])])
dc.MAX_PRINTABLE = 5
token = DataToken('a', '1', ('a', 0))
dc.set_token(token)
self.assertIsNotNone(dc.get_token())
self.assertEqual(dc.get_short_printable_repr(), '...a=1...')
def create_simple_fuzzable_request(unique_id):
unique_id = str(unique_id)
url = URL('http://w3af.com/')
headers = Headers([(u'Hello', u'World')])
post_data = KeyValueContainer(init_val=[('a', [unique_id])])
return FuzzableRequest(url,
method='GET',
post_data=post_data,
headers=headers)
def test_iter_setters(self):
dc = KeyValueContainer([(u'a', ['1']), (u'b', ['2', '3'])])
kv_setter = [(k, v, p, s) for (k, v, p, s) in dc.iter_setters()]
EXPECTED_KEY_VALUES = [('a', '1', ('a', 0)), ('b', '2', ('b', 0)),
('b', '3', ('b', 1))]
kvp = [(key, value, path) for (key, value, path, _) in kv_setter]
self.assertEqual(EXPECTED_KEY_VALUES, kvp)
for idx, (key, value, path, setter) in enumerate(kv_setter):
if idx == 2:
setter('w')
self.assertEqual(str(dc), 'a=1&b=2&b=w')
SET_VALUES = ['x', 'y', 'z']
for idx, (key, value, path, setter) in enumerate(kv_setter):
setter(SET_VALUES[idx])
self.assertEqual(str(dc), 'a=x&b=y&b=z')
def test_iter_setters(self):
dc = KeyValueContainer([(u'a', ['1']), (u'b', ['2', '3'])])
kv_setter = [(k, v, p, s) for (k, v, p, s) in dc.iter_setters()]
EXPECTED_KEY_VALUES = [('a', '1', ('a', 0)),
('b', '2', ('b', 0)),
('b', '3', ('b', 1))]
kvp = [(key, value, path) for (key, value, path, _) in kv_setter]
self.assertEqual(EXPECTED_KEY_VALUES, kvp)
for idx, (key, value, path, setter) in enumerate(kv_setter):
if idx == 2:
setter('w')
self.assertEqual(str(dc), 'a=1&b=2&b=w')
SET_VALUES = ['x', 'y', 'z']
for idx, (key, value, path, setter) in enumerate(kv_setter):
setter(SET_VALUES[idx])
self.assertEqual(str(dc), 'a=x&b=y&b=z')
def test_dump_case02(self):
expected = u'\r\n'.join([u'GET http://w3af.com/a/b/c.php HTTP/1.1',
u'Hola: Múndo',
u'',
u'a=b'])
headers = Headers([(u'Hola', u'Múndo')])
post_data = KeyValueContainer(init_val=[('a', ['b'])])
fr = FuzzableRequest(self.url, method='GET', post_data=post_data,
headers=headers)
self.assertEqual(fr.dump(), expected.encode('utf-8'))
def test_get_short_printable_repr_token_obj(self):
dc = KeyValueContainer([(u'a', ['1']), (u'b', ['2', '3'])])
token = DataToken('a', '1', ('a', 0))
dc.set_token(token)
self.assertIsNotNone(dc.get_token())
self.assertEqual(dc.get_short_printable_repr(), 'a=1&b=2&b=3')
def test_get_short_printable_repr_unicode_value_unicode(self):
dc = KeyValueContainer([(u'aéb', [u'céd']), (u'b', ['2', '3'])])
dc.MAX_PRINTABLE = 7
token = DataToken(u'aéb', u'céd', (u'aéb', 0))
dc.set_token(token)
self.assertEqual(dc.get_short_printable_repr(), '...a.b=c.d...')
def test_get_short_printable_repr_unicode_value(self):
dc = KeyValueContainer([(u'a', ['x']), (u'b', ['2', '3'])])
dc.MAX_PRINTABLE = 5
token = DataToken('a', 'é', ('a', 0))
dc.set_token(token)
self.assertEqual(dc.get_short_printable_repr(), '...a=....')
def test_set_token_using_data_token(self):
dc = KeyValueContainer([(u'a', ['1']), (u'b', ['2', '3'])])
token = DataToken('a', '1', ('a', 0))
set_token = dc.set_token(token)
self.assertEqual(dc.get_token().get_name(), 'a')
self.assertEqual(dc.get_token().get_value(), '1')
self.assertEqual(dc.get_token().get_path(), ('a', 0))
self.assertIs(dc.get_token(), token)
self.assertIs(set_token, token)
def test_get_short_printable_repr_token_obj_reduce_printable(self):
dc = KeyValueContainer([(u'a', ['1']), (u'b', ['2', '3'])])
dc.MAX_PRINTABLE = 5
token = DataToken('a', '1', ('a', 0))
dc.set_token(token)
self.assertIsNotNone(dc.get_token())
self.assertEqual(dc.get_short_printable_repr(), '...a=1...')
def test_dump_case03(self):
header_value = ''.join(chr(i) for i in xrange(256))
expected = u'\r\n'.join([u'GET http://w3af.com/a/b/c.php HTTP/1.1',
u'Hola: %s' % smart_unicode(header_value),
u'',
u'a=b'])
headers = Headers([(u'Hola', header_value)])
post_data = KeyValueContainer(init_val=[('a', ['b'])])
fr = FuzzableRequest(self.url, method='GET', post_data=post_data,
headers=headers)
self.assertEqual(fr.dump(), expected)
def test_copy_with_token(self):
dc = KeyValueContainer([(u'a', ['1']), (u'b', ['2', '3'])])
dc.set_token(('a', 0))
dc_copy = copy.deepcopy(dc)
self.assertEqual(dc.get_token(), dc_copy.get_token())
self.assertIsNotNone(dc.get_token())
self.assertIsNotNone(dc_copy.get_token())
def test_set_token_using_data_token(self):
dc = KeyValueContainer([(u'a', ['1']), (u'b', ['2', '3'])])
token = DataToken('a', '1', ('a', 0))
set_token = dc.set_token(token)
self.assertEqual(dc.get_token().get_name(), 'a')
self.assertEqual(dc.get_token().get_value(), '1')
self.assertEqual(dc.get_token().get_path(), ('a', 0))
self.assertIs(dc.get_token(), token)
self.assertIs(set_token, token)
def test_dump_mangle(self):
fr = FuzzableRequest(URL('http://www.w3af.com/'),
headers=Headers([('Host', 'www.w3af.com')]))
expected = u'\r\n'.join([
u'GET http://www.w3af.com/ HTTP/1.1', u'Host: www.w3af.com', u'',
u''
])
self.assertEqual(fr.dump(), expected)
fr.set_method('POST')
fr.set_data(KeyValueContainer(init_val=[('data', ['23'])]))
expected = u'\r\n'.join([
u'POST http://www.w3af.com/ HTTP/1.1', u'Host: www.w3af.com', u'',
u'data=23'
])
self.assertEqual(fr.dump(), expected)
def test_get_short_printable_repr_no_token(self):
dc = KeyValueContainer([(u'a', ['1']), (u'b', ['2', '3'])])
self.assertEqual(dc.get_short_printable_repr(), 'a=1&b=2&b=3')
class FuzzableRequest(RequestMixIn, DiskItem):
"""
This class represents a fuzzable request. Fuzzable requests were created
to allow w3af plugins to be much simpler and don't really care if the
vulnerability is in the postdata, querystring, header, cookie or any other
injection point.
FuzzableRequest classes are just an easy to use representation of an HTTP
Request, which will (during the audit phase) be wrapped into a Mutant
and have its values modified.
:author: Andres Riancho ([email protected])
"""
# In most cases we don't care about these headers, even if provided by the
# user, since they will be calculated based on the attributes we are
# going to store and these won't be updated.
REMOVE_HEADERS = ('content-length',)
__slots__ = ('_method',
'_cookie',
'_post_data',
'_headers',
'_uri',
'_url',
'_sent_info_comp')
def __init__(self, uri, method='GET', headers=None, cookie=None,
post_data=None):
super(FuzzableRequest, self).__init__()
# Note: Do not check for the URI/Headers type here, since I'm doing it
# in set_uri() and set_headers() already.
if cookie is not None and not isinstance(cookie, Cookie):
raise TypeError(TYPE_ERROR % ('cookie', 'Cookie'))
if post_data is not None and not isinstance(post_data, DataContainer):
raise TypeError(TYPE_ERROR % ('post_data', 'DataContainer'))
# Internal variables
self._method = method
self._cookie = Cookie() if cookie is None else cookie
self._post_data = KeyValueContainer() if post_data is None else post_data
# Set the headers
self._headers = None
pheaders = Headers() if headers is None else headers
self.set_headers(pheaders)
# Set the URL
self._uri = None
self._url = None
self.set_uri(uri)
# Set the internal variables
self._sent_info_comp = None
def __getstate__(self):
state = {k: getattr(self, k) for k in self.__slots__}
return state
def __setstate__(self, state):
[setattr(self, k, v) for k, v in state.iteritems()]
def get_default_headers(self):
"""
:return: The headers we want to use framework-wide for fuzzing. By
default we set the fuzzable_headers to [], which makes this
method return an empty Headers instance.
When the user sets a fuzzable_headers it will create a Headers
instance with empty values.
We then append the specific headers supplied for this
FuzzableRequest instance to the default headers. Any specific
headers override the default (empty) ones.
"""
fuzzable_headers = cf.cf.get('fuzzable_headers') or []
req_headers = [(h, '') for h in fuzzable_headers]
return Headers(init_val=req_headers)
@classmethod
def from_parts(cls, url, method='GET', post_data=None, headers=None):
"""
:return: An instance of FuzzableRequest from the provided parameters.
"""
if isinstance(url, basestring):
url = URL(url)
if post_data == '':
post_data = None
elif isinstance(post_data, basestring):
post_data = dc_from_hdrs_post(headers, post_data)
return cls(url, method=method, headers=headers, post_data=post_data)
@classmethod
def from_http_response(cls, http_response):
"""
:return: An instance of FuzzableRequest using the URL and cookie from
the http_response. The method used is "GET", and no post_data
is set.
"""
cookie = Cookie.from_http_response(http_response)
return cls(http_response.get_uri(), method='GET', cookie=cookie)
@classmethod
def from_http_request(cls, request):
"""
:param request: The instance we'll use as base
:return: An instance of FuzzableRequest based on a urllib2 HTTP request
instance.
"""
headers = request.headers
headers.update(request.unredirected_hdrs)
headers = Headers(headers.items())
post_data = request.get_data() or ''
return cls.from_parts(request.url_object, method=request.get_method(),
headers=headers, post_data=post_data)
@classmethod
def from_form(cls, form, headers=None):
if form.get_method().upper() == 'POST':
r = cls(form.get_action(),
method=form.get_method(),
headers=headers,
post_data=form)
else:
# The default is a GET request
form_action = form.get_action()
form_action.querystring = form
r = cls(form_action,
method=form.get_method(),
headers=headers)
return r
def to_base64(self):
"""
:return: The whole HTTP request serialized and encoded as base64
"""
raw_http_request = self.dump()
return base64.b64encode(raw_http_request)
@classmethod
def from_base64(cls, base64_data):
"""
:param base64_data: A string generated by to_base64
:return: A FuzzableRequest instance
"""
from w3af.core.data.parsers.doc.http_request_parser import raw_http_request_parser
raw_http_request = base64.b64decode(base64_data)
return raw_http_request_parser(raw_http_request)
def make_comp(self, heterogen_string):
"""
This basically removes characters that are used as escapes such as \
"""
return string.translate(heterogen_string, TRANS_TABLE,
deletions=DELETE_CHARS)
def sent(self, needle):
"""
Checks if something similar to `needle` was sent in the request.
This is used to remove false positives, e.g. if a grep plugin finds a
"strange" string and wants to be sure it was not generated by an audit
plugin.
This method should only be used by grep plugins which often have false
positives.
The following example shows that we sent d'z"0 but d\'z"0 will
as well be recognised as sent
Note on performance:
At some point I thought about making all these calls lazy:
needles.add(unquote(needle))
needles.add(quote(needle))
needles.add(quote_plus(needle))
needles.add(self.make_comp(needle))
needles.add(self.make_comp(unquote(needle)))
To avoid the potentially unnecessary call to self.make_comp(...)
if the needle was found in a haystack before, making the result
of self.make_comp(...) unnecessary.
That would help, but the impact in real life is really small, since
in most scenarios this method will return False, which means that
all the comparisons need to be done anyways.
:param needle: The string
:return: True if something similar was sent
"""
needle = smart_str_ignore(needle)
needles = set()
needles.add(needle)
needles.add(unquote(needle))
needles.add(quote(needle))
needles.add(quote_plus(needle))
needles.add(self.make_comp(needle))
needles.add(self.make_comp(unquote(needle)))
# Filter the short needles
#
# We don't want false negatives just because the string is
# short after making comparable
needles = {n for n in needles if len(n) >= 3}
uri = self.get_uri()
data = smart_str_ignore(self.get_data())
headers = smart_str_ignore(self.get_all_headers())
haystacks = set()
haystacks.add(smart_str_ignore(uri))
haystacks.add(smart_str_ignore(uri.url_decode()))
haystacks.add(self.make_comp(smart_str_ignore(uri.url_decode())))
haystacks.add(data)
haystacks.add(unquote(data))
haystacks.add(self.make_comp(data))
haystacks.add(self.make_comp(unquote(data)))
haystacks.add(headers)
haystacks.add(unquote(headers))
# Filter the short haystacks
haystacks = {h for h in haystacks if len(h) >= 3}
for needle in needles:
for haystack in haystacks:
if needle in haystack:
return True
# I didn't send the needle in any way
return False
def __hash__(self):
return hash(str(self.get_uri()) + self.get_data())
def __str__(self):
"""
:return: A string representation of this fuzzable request.
"""
short_fmt = u'Method: %s | %s'
long_fmt = u'Method: %s | %s | %s: (%s)'
if self.get_raw_data():
parameters = self.get_raw_data().get_param_names()
dc_type = self.get_raw_data().get_type()
else:
parameters = self.get_uri().querystring.get_param_names()
dc_type = self.get_uri().querystring.get_type()
if not parameters:
output = short_fmt % (self.get_method(), self.get_url())
else:
jparams = u', '.join(parameters)
output = long_fmt % (self.get_method(), self.get_url(),
dc_type, jparams)
return output.encode(DEFAULT_ENCODING)
def __unicode__(self):
return str(self).decode(encoding=DEFAULT_ENCODING, errors='ignore')
def __repr__(self):
return '<fuzzable request | %s | %s>' % (self.get_method(),
self.get_uri())
def __eq__(self, other):
"""
Two requests are equal if:
- They have the same URL
- They have the same method
- They have the same parameters
- The values for each parameter is equal
:return: True if the requests are equal.
"""
if isinstance(other, FuzzableRequest):
return (self.get_method() == other.get_method() and
self.get_uri() == other.get_uri() and
self.get_raw_data() == other.get_raw_data() and
self.get_headers() == other.get_headers())
return False
def get_eq_attrs(self):
return ['_method', '_uri', '_post_data', '_headers']
def __ne__(self, other):
return not self.__eq__(other)
def is_variant_of(self, other):
"""
Two requests are loosely equal (or variants) if:
- They have the same URL
- They have the same HTTP method
- They have the same parameter names
- The values for each parameter have the same type (int / string)
:return: True if self and other are variants.
"""
if self.get_method() != other.get_method():
return False
if self.get_url() != other.get_url():
return False
self_qs = self.get_uri().querystring
other_qs = other.get_uri().querystring
if not self_qs.is_variant_of(other_qs):
return False
return True
def set_url(self, url):
if not isinstance(url, URL):
msg = 'The "uri" parameter of a %s must be of url.URL type.'
raise TypeError(msg % type(self).__name__)
self._url = URL(url.url_string.replace(' ', '%20'))
self._uri = self._url
def set_uri(self, uri):
if not isinstance(uri, URL):
msg = 'The "uri" parameter of a %s must be of url.URL type.'
raise TypeError(msg % type(self).__name__)
self._uri = uri
self._url = uri.uri2url()
def get_querystring(self):
return self.get_uri().querystring
def set_querystring(self, new_qs):
self.get_uri().querystring = new_qs
def set_method(self, method):
self._method = method
def set_headers(self, headers):
if headers is not None and not isinstance(headers, Headers):
raise TypeError(TYPE_ERROR % ('headers', 'Headers'))
for header_name in self.REMOVE_HEADERS:
try:
headers.idel(header_name)
except KeyError:
# We don't care if they don't exist
pass
for k, v in self.get_default_headers().items():
# Ignore any keys which are already defined in the user-specified
# headers
kvalue, kreal = headers.iget(k, None)
if kvalue is not None:
continue
headers[k] = v
self._headers = headers
def set_referer(self, referer):
self._headers['Referer'] = str(referer)
def set_cookie(self, cookie):
"""
:param cookie: A Cookie object as defined in core.data.dc.cookie,
or a string.
"""
if isinstance(cookie, Cookie):
self._cookie = cookie
elif isinstance(cookie, basestring):
self._cookie = Cookie(cookie)
elif cookie is None:
self._cookie = Cookie()
else:
fmt = '[FuzzableRequest error] set_cookie received: "%s": "%s".'
error_str = fmt % (type(cookie), repr(cookie))
om.out.error(error_str)
raise BaseFrameworkException(error_str)
def get_url(self):
return self._url
def get_uri(self):
"""
:return: The URI to send in the HTTP request
"""
return self._uri
def set_data(self, post_data):
"""
Set the DataContainer which we'll use for post-data
"""
if not isinstance(post_data, DataContainer):
raise TypeError('The "post_data" parameter of a %s must be of '
'DataContainer type.' % type(self).__name__)
self._post_data = post_data
def get_data(self):
"""
The data is the string representation of the post data, in most
cases it will be used as the POSTDATA for requests.
"""
return str(self._post_data)
def get_raw_data(self):
return self._post_data
def get_method(self):
return self._method
def get_post_data_headers(self):
"""
:return: A Headers object with the headers required to send the
self._post_data to the wire. For example, if the data is
url-encoded:
a=3&b=2
This method returns:
Content-Length: 7
Content-Type: application/x-www-form-urlencoded
When someone queries this object for the headers using
get_headers(), we'll include these. Hopefully this means that
the required headers will make it to the wire.
"""
return Headers(init_val=self.get_raw_data().get_headers())
def get_headers(self):
"""
:return: The headers which can be changed by the user during fuzzing.
:see: get_all_headers
"""
return self._headers
def get_all_headers(self):
"""
:return: Calls get_default_headers to get the default framework headers,
get_post_data_headers to get the DataContainer headers, merges that info
with the user specified headers (which live in self._headers) and
returns a Headers instance which will be sent to the wire.
"""
wire_headers = Headers()
for k, v in chain(self._headers.items(),
self.get_post_data_headers().items()):
# Please note that here we're overwriting the headers from the
# fuzzable request with the headers from the data container,
# the overwriting is done in this order due to the order in the
# chain() items above
#
# I found a bug where I loaded a request from spider_man, saved
# it using dump() and then tried to load it again and failed because
# of this overwriting not being done (the multipart boundary was
# incorrect).
#
# Keep that in mind in case you want to change this overwriting!
#
# Overwrite the existing one, case insensitive style
_, stored_header_name = wire_headers.iget(k, None)
if stored_header_name is not None:
wire_headers[stored_header_name] = v
else:
wire_headers[k] = v
return wire_headers
def get_referer(self):
return self.get_headers().get('Referer', None)
def get_cookie(self):
return self._cookie
def get_file_vars(self):
"""
:return: A list of post-data parameters that contain a file
"""
try:
return self._post_data.get_file_vars()
except AttributeError:
return []
def test_is_variant_of_eq_keys_neq_value_types(self):
dc1 = KeyValueContainer([(u'a', ['1']), (u'b', ['c', '3'])])
dc2 = KeyValueContainer([(u'a', ['1']), (u'b', ['2', '3'])])
self.assertFalse(dc1.is_variant_of(dc2))
def test_set_token(self):
dc = KeyValueContainer([(u'a', ['1']), (u'b', ['2', '3'])])
token = dc.set_token(('a', 0))
self.assertEqual(token.get_name(), 'a')
self.assertEqual(token, dc['a'][0])
def test_double_data_token_wrap_set_set(self):
dc = KeyValueContainer([(u'a', ['1']), (u'b', ['c', '3'])])
token_1 = dc.set_token(('b', 1))
token_2 = dc.set_token(('b', 1))
self.assertIs(token_1, token_2)
def test_double_data_token_wrap_set_set(self):
dc = KeyValueContainer([(u'a', ['1']), (u'b', ['c', '3'])])
token_1 = dc.set_token(('b', 1))
token_2 = dc.set_token(('b', 1))
self.assertIs(token_1, token_2)
class FuzzableRequest(RequestMixIn, DiskItem):
"""
This class represents a fuzzable request. Fuzzable requests were created
to allow w3af plugins to be much simpler and don't really care if the
vulnerability is in the postdata, querystring, header, cookie or any other
injection point.
FuzzableRequest classes are just an easy to use representation of an HTTP
Request, which will (during the audit phase) be wrapped into a Mutant
and have its values modified.
:author: Andres Riancho ([email protected])
"""
# In most cases we don't care about these headers, even if provided by the
# user, since they will be calculated based on the attributes we are
# going to store and these won't be updated.
REMOVE_HEADERS = ('content-length', )
__slots__ = ('_method', '_cookie', '_post_data', '_headers', '_uri',
'_url', '_sent_info_comp')
def __init__(self,
uri,
method='GET',
headers=None,
cookie=None,
post_data=None):
super(FuzzableRequest, self).__init__()
# Note: Do not check for the URI/Headers type here, since I'm doing it
# in set_uri() and set_headers() already.
if cookie is not None and not isinstance(cookie, Cookie):
raise TypeError(TYPE_ERROR % ('cookie', 'Cookie'))
if post_data is not None and not isinstance(post_data, DataContainer):
raise TypeError(TYPE_ERROR % ('post_data', 'DataContainer'))
# Internal variables
self._method = method
self._cookie = Cookie() if cookie is None else cookie
self._post_data = KeyValueContainer(
) if post_data is None else post_data
# Set the headers
self._headers = None
pheaders = Headers() if headers is None else headers
self.set_headers(pheaders)
# Set the URL
self._uri = None
self._url = None
self.set_uri(uri)
# Set the internal variables
self._sent_info_comp = None
def __getstate__(self):
state = {k: getattr(self, k) for k in self.__slots__}
return state
def __setstate__(self, state):
[setattr(self, k, v) for k, v in state.iteritems()]
def get_default_headers(self):
"""
:return: The headers we want to use framework-wide for fuzzing. By
default we set the fuzzable_headers to [], which makes this
method return an empty Headers instance.
When the user sets a fuzzable_headers it will create a Headers
instance with empty values.
We then append the specific headers supplied for this
FuzzableRequest instance to the default headers. Any specific
headers override the default (empty) ones.
"""
fuzzable_headers = cf.cf.get('fuzzable_headers') or []
req_headers = [(h, '') for h in fuzzable_headers]
return Headers(init_val=req_headers)
@classmethod
def from_parts(cls, url, method='GET', post_data=None, headers=None):
"""
:return: An instance of FuzzableRequest from the provided parameters.
"""
if isinstance(url, basestring):
url = URL(url)
if post_data == '':
post_data = None
elif isinstance(post_data, basestring):
post_data = dc_from_hdrs_post(headers, post_data)
return cls(url, method=method, headers=headers, post_data=post_data)
@classmethod
def from_http_response(cls, http_response):
"""
:return: An instance of FuzzableRequest using the URL and cookie from
the http_response. The method used is "GET", and no post_data
is set.
"""
cookie = Cookie.from_http_response(http_response)
return cls(http_response.get_uri(), method='GET', cookie=cookie)
@classmethod
def from_http_request(cls, request):
"""
:param request: The instance we'll use as base
:return: An instance of FuzzableRequest based on a urllib2 HTTP request
instance.
"""
headers = request.headers
headers.update(request.unredirected_hdrs)
headers = Headers(headers.items())
post_data = request.get_data() or ''
return cls.from_parts(request.url_object,
method=request.get_method(),
headers=headers,
post_data=post_data)
@classmethod
def from_form(cls, form, headers=None):
if form.get_method().upper() == 'POST':
r = cls(form.get_action(),
method=form.get_method(),
headers=headers,
post_data=form)
else:
# The default is a GET request
form_action = form.get_action()
form_action.querystring = form
r = cls(form_action, method=form.get_method(), headers=headers)
return r
def to_base64(self):
"""
:return: The whole HTTP request serialized and encoded as base64
"""
raw_http_request = self.dump()
return base64.b64encode(raw_http_request)
@classmethod
def from_base64(cls, base64_data):
"""
:param base64_data: A string generated by to_base64
:return: A FuzzableRequest instance
"""
from w3af.core.data.parsers.doc.http_request_parser import raw_http_request_parser
raw_http_request = base64.b64decode(base64_data)
return raw_http_request_parser(raw_http_request)
def make_comp(self, heterogen_string):
"""
This basically removes characters that are used as escapes such as \
"""
return string.translate(heterogen_string,
TRANS_TABLE,
deletions=DELETE_CHARS)
def sent(self, needle):
"""
Checks if something similar to `needle` was sent in the request.
This is used to remove false positives, e.g. if a grep plugin finds a
"strange" string and wants to be sure it was not generated by an audit
plugin.
This method should only be used by grep plugins which often have false
positives.
The following example shows that we sent d'z"0 but d\'z"0 will
as well be recognised as sent
Note on performance:
At some point I thought about making all these calls lazy:
needles.add(unquote(needle))
needles.add(quote(needle))
needles.add(quote_plus(needle))
needles.add(self.make_comp(needle))
needles.add(self.make_comp(unquote(needle)))
To avoid the potentially unnecessary call to self.make_comp(...)
if the needle was found in a haystack before, making the result
of self.make_comp(...) unnecessary.
That would help, but the impact in real life is really small, since
in most scenarios this method will return False, which means that
all the comparisons need to be done anyways.
:param needle: The string
:return: True if something similar was sent
"""
needle = smart_str_ignore(needle)
needles = set()
needles.add(needle)
needles.add(unquote(needle))
needles.add(quote(needle))
needles.add(quote_plus(needle))
needles.add(self.make_comp(needle))
needles.add(self.make_comp(unquote(needle)))
# Filter the short needles
#
# We don't want false negatives just because the string is
# short after making comparable
needles = {n for n in needles if len(n) >= 3}
uri = self.get_uri()
data = smart_str_ignore(self.get_data())
headers = smart_str_ignore(self.get_all_headers())
haystacks = set()
haystacks.add(smart_str_ignore(uri))
haystacks.add(smart_str_ignore(uri.url_decode()))
haystacks.add(self.make_comp(smart_str_ignore(uri.url_decode())))
haystacks.add(data)
haystacks.add(unquote(data))
haystacks.add(self.make_comp(data))
haystacks.add(self.make_comp(unquote(data)))
haystacks.add(headers)
haystacks.add(unquote(headers))
# Filter the short haystacks
haystacks = {h for h in haystacks if len(h) >= 3}
haystack = '--'.join(haystacks)
for needle in needles:
if needle in haystack:
return True
# I didn't send the needle in any way
return False
def get_hash(self):
raw_http_request = self.dump()
return hashlib.md5(raw_http_request).hexdigest()
def __hash__(self):
return hash(str(self.get_uri()) + self.get_data())
def __str__(self):
"""
:return: A string representation of this fuzzable request.
"""
short_fmt = u'Method: %s | %s'
long_fmt = u'Method: %s | %s | %s: (%s)'
if self.get_raw_data():
parameters = self.get_raw_data().get_param_names()
dc_type = self.get_raw_data().get_type()
else:
parameters = self.get_uri().querystring.get_param_names()
dc_type = self.get_uri().querystring.get_type()
if not parameters:
output = short_fmt % (self.get_method(), self.get_url())
else:
jparams = u', '.join(parameters)
output = long_fmt % (self.get_method(), self.get_url(), dc_type,
jparams)
return output.encode(DEFAULT_ENCODING)
def __unicode__(self):
return str(self).decode(encoding=DEFAULT_ENCODING, errors='ignore')
def __repr__(self):
return '<fuzzable request | %s | %s>' % (self.get_method(),
self.get_uri())
def __eq__(self, other):
"""
Two requests are equal if:
- They have the same URL
- They have the same method
- They have the same parameters
- The values for each parameter is equal
:return: True if the requests are equal.
"""
if isinstance(other, FuzzableRequest):
return (self.get_method() == other.get_method()
and self.get_uri() == other.get_uri()
and self.get_raw_data() == other.get_raw_data()
and self.get_headers() == other.get_headers())
return False
def get_eq_attrs(self):
return ['_method', '_uri', '_post_data', '_headers']
def __ne__(self, other):
return not self.__eq__(other)
def is_variant_of(self, other):
"""
Two requests are loosely equal (or variants) if:
- They have the same URL
- They have the same HTTP method
- They have the same parameter names
- The values for each parameter have the same type (int / string)
:return: True if self and other are variants.
"""
if self.get_method() != other.get_method():
return False
if self.get_url() != other.get_url():
return False
self_qs = self.get_uri().querystring
other_qs = other.get_uri().querystring
if not self_qs.is_variant_of(other_qs):
return False
return True
def set_url(self, url):
if not isinstance(url, URL):
msg = 'The "uri" parameter of a %s must be of url.URL type.'
raise TypeError(msg % type(self).__name__)
self._url = URL(url.url_string.replace(' ', '%20'))
self._uri = self._url
def set_uri(self, uri):
if not isinstance(uri, URL):
msg = 'The "uri" parameter of a %s must be of url.URL type.'
raise TypeError(msg % type(self).__name__)
self._uri = uri
self._url = uri.uri2url()
def get_querystring(self):
return self.get_uri().querystring
def set_querystring(self, new_qs):
self.get_uri().querystring = new_qs
def set_method(self, method):
self._method = method
def set_headers(self, headers):
if headers is not None and not isinstance(headers, Headers):
raise TypeError(TYPE_ERROR % ('headers', 'Headers'))
for header_name in self.REMOVE_HEADERS:
try:
headers.idel(header_name)
except KeyError:
# We don't care if they don't exist
pass
for k, v in self.get_default_headers().items():
# Ignore any keys which are already defined in the user-specified
# headers
kvalue, kreal = headers.iget(k, None)
if kvalue is not None:
continue
headers[k] = v
self._headers = headers
def set_referer(self, referer):
self._headers['Referer'] = str(referer)
def set_cookie(self, cookie):
"""
:param cookie: A Cookie object as defined in core.data.dc.cookie,
or a string.
"""
if isinstance(cookie, Cookie):
self._cookie = cookie
elif isinstance(cookie, basestring):
self._cookie = Cookie(cookie)
elif cookie is None:
self._cookie = Cookie()
else:
fmt = '[FuzzableRequest error] set_cookie received: "%s": "%s".'
error_str = fmt % (type(cookie), repr(cookie))
om.out.error(error_str)
raise BaseFrameworkException(error_str)
def get_url(self):
return self._url
def get_uri(self):
"""
:return: The URI to send in the HTTP request
"""
return self._uri
def set_data(self, post_data):
"""
Set the DataContainer which we'll use for post-data
"""
if not isinstance(post_data, DataContainer):
raise TypeError('The "post_data" parameter of a %s must be of '
'DataContainer type.' % type(self).__name__)
self._post_data = post_data
def get_data(self):
"""
The data is the string representation of the post data, in most
cases it will be used as the POSTDATA for requests.
"""
return str(self._post_data)
def get_raw_data(self):
return self._post_data
def get_method(self):
return self._method
def get_post_data_headers(self):
"""
:return: A Headers object with the headers required to send the
self._post_data to the wire. For example, if the data is
url-encoded:
a=3&b=2
This method returns:
Content-Length: 7
Content-Type: application/x-www-form-urlencoded
When someone queries this object for the headers using
get_headers(), we'll include these. Hopefully this means that
the required headers will make it to the wire.
"""
return Headers(init_val=self.get_raw_data().get_headers())
def get_headers(self):
"""
:return: The headers which can be changed by the user during fuzzing.
:see: get_all_headers
"""
return self._headers
def get_all_headers(self):
"""
:return: Calls get_default_headers to get the default framework headers,
get_post_data_headers to get the DataContainer headers, merges that info
with the user specified headers (which live in self._headers) and
returns a Headers instance which will be sent to the wire.
"""
wire_headers = Headers()
for k, v in chain(self._headers.items(),
self.get_post_data_headers().items()):
# Please note that here we're overwriting the headers from the
# fuzzable request with the headers from the data container,
# the overwriting is done in this order due to the order in the
# chain() items above
#
# I found a bug where I loaded a request from spider_man, saved
# it using dump() and then tried to load it again and failed because
# of this overwriting not being done (the multipart boundary was
# incorrect).
#
# Keep that in mind in case you want to change this overwriting!
#
# Overwrite the existing one, case insensitive style
_, stored_header_name = wire_headers.iget(k, None)
if stored_header_name is not None:
wire_headers[stored_header_name] = v
else:
wire_headers[k] = v
return wire_headers
def get_referer(self):
return self.get_headers().get('Referer', None)
def get_cookie(self):
return self._cookie
def get_file_vars(self):
"""
:return: A list of post-data parameters that contain a file
"""
try:
return self._post_data.get_file_vars()
except AttributeError:
return []
def test_str_special_chars(self):
dc = KeyValueContainer([(u'a', ['1']), (u'u', [u'Ú-ú-Ü-ü'])], 'latin1')
decoded_str = urllib.unquote(str(dc)).decode('latin-1')
self.assertEquals(u'a=1&u=Ú-ú-Ü-ü', decoded_str)
def test_set_token(self):
dc = KeyValueContainer([(u'a', ['1']), (u'b', ['2', '3'])])
token = dc.set_token(('a', 0))
self.assertEqual(token.get_name(), 'a')
self.assertEqual(token, dc['a'][0])
def test_unicode(self):
dc = KeyValueContainer([(u'a', ['1']), (u'b', ['2', u'3'])])
udc = unicode(dc)
self.assertEqual(udc, u'a=1&b=2&b=3')
self.assertIsInstance(udc, unicode)
def test_export_import_with_post_data(self):
dc = KeyValueContainer(init_val=[('a', ['1'])])
fr = FuzzableRequest(URL('http://www.w3af.com/'), post_data=dc)
imported_fr = FuzzableRequest.from_base64(fr.to_base64())
self.assertEqual(imported_fr, fr)
class FuzzableRequest(RequestMixIn, DiskItem):
"""
This class represents a fuzzable request. Fuzzable requests were created
to allow w3af plugins to be much simpler and don't really care if the
vulnerability is in the postdata, querystring, header, cookie or any other
injection point.
FuzzableRequest classes are just an easy to use representation of an HTTP
Request, which will (during the audit phase) be wrapped into a Mutant
and have its values modified.
:author: Andres Riancho ([email protected])
"""
# In most cases we don't care about these headers, even if provided by the
# user, since they will be calculated based on the attributes we are
# going to store and these won't be updated.
REMOVE_HEADERS = ('content-length',)
__slots__ = ('_method',
'_cookie',
'_post_data',
'_headers',
'_uri',
'_url',
'_sent_info_comp')
def __init__(self, uri, method='GET', headers=None, cookie=None,
post_data=None):
super(FuzzableRequest, self).__init__()
# Note: Do not check for the URI/Headers type here, since I'm doing it
# in set_uri() and set_headers() already.
if cookie is not None and not isinstance(cookie, Cookie):
raise TypeError(TYPE_ERROR % ('cookie', 'Cookie'))
if post_data is not None and not isinstance(post_data, DataContainer):
raise TypeError(TYPE_ERROR % ('post_data', 'DataContainer'))
# Internal variables
self._method = method
self._cookie = Cookie() if cookie is None else cookie
self._post_data = KeyValueContainer() if post_data is None else post_data
# Set the headers
self._headers = None
pheaders = Headers() if headers is None else headers
self.set_headers(pheaders)
# Set the URL
self._uri = None
self._url = None
self.set_uri(uri)
# Set the internal variables
self._sent_info_comp = None
def __getstate__(self):
state = {k: getattr(self, k) for k in self.__slots__}
return state
def __setstate__(self, state):
[setattr(self, k, v) for k, v in state.iteritems()]
def get_default_headers(self):
"""
:return: The headers we want to use framework-wide for fuzzing. By
default we set the fuzzable_headers to [], which makes this
method return an empty Headers instance.
When the user sets a fuzzable_headers it will create a Headers
instance with empty values.
We then append the specific headers supplied for this
FuzzableRequest instance to the default headers. Any specific
headers override the default (empty) ones.
"""
fuzzable_headers = cf.cf.get('fuzzable_headers') or []
req_headers = [(h, '') for h in fuzzable_headers]
return Headers(init_val=req_headers)
@classmethod
def from_parts(cls, url, method='GET', post_data=None, headers=None):
"""
:return: An instance of FuzzableRequest from the provided parameters.
"""
if isinstance(url, basestring):
url = URL(url)
if post_data == '':
post_data = None
elif isinstance(post_data, basestring):
post_data = dc_from_hdrs_post(headers, post_data)
return cls(url, method=method, headers=headers, post_data=post_data)
@classmethod
def from_http_response(cls, http_response):
"""
:return: An instance of FuzzableRequest using the URL and cookie from
the http_response. The method used is "GET", and no post_data
is set.
"""
cookie = Cookie.from_http_response(http_response)
return cls(http_response.get_uri(), method='GET', cookie=cookie)
@classmethod
def from_urllib2_request(cls, request):
"""
:param request: The instance we'll use as base
:return: An instance of FuzzableRequest based on a urllib2 HTTP request
instance.
"""
headers = request.headers
headers.update(request.unredirected_hdrs)
headers = Headers(headers.items())
post_data = request.get_data() or ''
return cls.from_parts(request.url_object, method=request.get_method(),
headers=headers, post_data=post_data)
@classmethod
def from_form(cls, form, headers=None):
if form.get_method().upper() == 'POST':
r = cls(form.get_action(),
method=form.get_method(),
headers=headers,
post_data=form)
else:
# The default is a GET request
form_action = form.get_action()
form_action.querystring = form
r = cls(form_action,
method=form.get_method(),
headers=headers)
return r
def to_csv(self):
"""
Generic version of how fuzzable requests are exported:
METHOD,URL,POST_DATA
Example:
GET,http://localhost/index.php?abc=123&def=789,
POST,http://localhost/index.php,abc=123&def=789
:return: a csv str representation of the request
"""
#
# TODO: Why don't we export headers and cookies?
#
output = cStringIO.StringIO()
writer = csv.writer(output, quoting=csv.QUOTE_ALL)
writer.writerow((self.get_method(), self.get_uri(), self._post_data))
output.seek(0)
return str(output.read())[:-2]
@classmethod
def from_csv(cls, csv_line):
"""
:param csv_line: A string representing a line in a CSV file
:return: A FuzzableRequest instance
"""
row = csv.reader([csv_line], quoting=csv.QUOTE_ALL).next()
return FuzzableRequest.from_parts(row[1], method=row[0],
post_data=row[2])
def sent(self, smth_instng):
"""
Checks if something similar to `smth_instng` was sent in the request.
This is used to remove false positives, e.g. if a grep plugin finds a
"strange" string and wants to be sure it was not generated by an audit
plugin.
This method should only be used by grep plugins which often have false
positives.
The following example shows that we sent d'z"0 but d\'z"0 will
as well be recognised as sent
TODO: This function is called MANY times, and under some circumstances
it's performance REALLY matters. We need to review this function.
:param smth_instng: The string
:return: True if something similar was sent
"""
def make_comp(heterogen_string):
"""
This basically removes characters that are hard to compare
"""
return string.translate(heterogen_string.encode('utf-8'),
TRANS_TABLE, deletions=DELETE_CHARS)
data = self.get_data()
# This is the easy part. If it was exactly like this in the request
if data and smth_instng in data or \
smth_instng in self.get_uri() or \
smth_instng in unquote(data) or \
smth_instng in unicode(self.get_uri().url_decode()):
return True
# Ok, it's not in it but maybe something similar
# Let's set up something we can compare
if self._sent_info_comp is None:
data_encoding = self._post_data.encoding
post_data = str(self.get_data())
dec_post_data = unquote(post_data).decode(data_encoding)
data = u'%s%s%s' % (unicode(self.get_uri()), data, dec_post_data)
self._sent_info_comp = make_comp(data + unquote(data))
min_len = 3
# make the smth_instng comparable
smth_instng_comps = (make_comp(smth_instng),
make_comp(unquote(smth_instng)))
for smth_intstng_comp in smth_instng_comps:
# We don't want false negatives just because the string is
# short after making comparable
if smth_intstng_comp in self._sent_info_comp and \
len(smth_intstng_comp) >= min_len:
return True
# I didn't sent the smth_instng in any way
return False
def __hash__(self):
return hash(str(self.get_uri()) + self.get_data())
def __str__(self):
"""
:return: A string representation of this fuzzable request.
"""
short_fmt = u'Method: %s | %s'
long_fmt = u'Method: %s | %s | %s: (%s)'
if self.get_raw_data():
parameters = self.get_raw_data().get_param_names()
dc_type = self.get_raw_data().get_type()
else:
parameters = self.get_uri().querystring.get_param_names()
dc_type = self.get_uri().querystring.get_type()
if not parameters:
output = short_fmt % (self.get_method(), self.get_url())
else:
jparams = u', '.join(parameters)
output = long_fmt % (self.get_method(), self.get_url(),
dc_type, jparams)
return output.encode(DEFAULT_ENCODING)
def __unicode__(self):
return str(self).decode(encoding=DEFAULT_ENCODING, errors='ignore')
def __repr__(self):
return '<fuzzable request | %s | %s>' % (self.get_method(),
self.get_uri())
def __eq__(self, other):
"""
Two requests are equal if:
- They have the same URL
- They have the same method
- They have the same parameters
- The values for each parameter is equal
:return: True if the requests are equal.
"""
if isinstance(other, FuzzableRequest):
return (self.get_method() == other.get_method() and
self.get_uri() == other.get_uri() and
self.get_raw_data() == other.get_raw_data() and
self.get_headers() == other.get_headers())
return False
def get_eq_attrs(self):
return ['_method', '_uri', '_post_data', '_headers']
def __ne__(self, other):
return not self.__eq__(other)
def is_variant_of(self, other):
"""
Two requests are loosely equal (or variants) if:
- They have the same URL
- They have the same HTTP method
- They have the same parameter names
- The values for each parameter have the same type (int / string)
:return: True if self and other are variants.
"""
if self.get_method() != other.get_method():
return False
if self.get_url() != other.get_url():
return False
self_qs = self.get_uri().querystring
other_qs = other.get_uri().querystring
if not self_qs.is_variant_of(other_qs):
return False
return True
def set_url(self, url):
if not isinstance(url, URL):
msg = 'The "uri" parameter of a %s must be of url.URL type.'
raise TypeError(msg % type(self).__name__)
self._url = URL(url.url_string.replace(' ', '%20'))
self._uri = self._url
def set_uri(self, uri):
if not isinstance(uri, URL):
msg = 'The "uri" parameter of a %s must be of url.URL type.'
raise TypeError(msg % type(self).__name__)
self._uri = uri
self._url = uri.uri2url()
def get_querystring(self):
return self.get_uri().querystring
def set_querystring(self, new_qs):
self.get_uri().querystring = new_qs
def set_method(self, method):
self._method = method
def set_headers(self, headers):
if headers is not None and not isinstance(headers, Headers):
raise TypeError(TYPE_ERROR % ('headers', 'Headers'))
for header_name in self.REMOVE_HEADERS:
try:
headers.idel(header_name)
except KeyError:
# We don't care if they don't exist
pass
for k, v in self.get_default_headers().items():
# Ignore any keys which are already defined in the user-specified
# headers
kvalue, kreal = headers.iget(k, None)
if kvalue is not None:
continue
headers[k] = v
self._headers = headers
def set_referer(self, referer):
self._headers['Referer'] = str(referer)
def set_cookie(self, cookie):
"""
:param cookie: A Cookie object as defined in core.data.dc.cookie,
or a string.
"""
if isinstance(cookie, Cookie):
self._cookie = cookie
elif isinstance(cookie, basestring):
self._cookie = Cookie(cookie)
elif cookie is None:
self._cookie = Cookie()
else:
fmt = '[FuzzableRequest error] set_cookie received: "%s": "%s".'
error_str = fmt % (type(cookie), repr(cookie))
om.out.error(error_str)
raise BaseFrameworkException(error_str)
def get_url(self):
return self._url
def get_uri(self):
"""
:return: The URI to send in the HTTP request
"""
return self._uri
def set_data(self, post_data):
"""
Set the DataContainer which we'll use for post-data
"""
if not isinstance(post_data, DataContainer):
raise TypeError('The "post_data" parameter of a %s must be of '
'DataContainer type.' % type(self).__name__)
self._post_data = post_data
def get_data(self):
"""
The data is the string representation of the post data, in most
cases it will be used as the POSTDATA for requests.
"""
return str(self._post_data)
def get_raw_data(self):
return self._post_data
def get_method(self):
return self._method
def get_post_data_headers(self):
"""
:return: A Headers object with the headers required to send the
self._post_data to the wire. For example, if the data is
url-encoded:
a=3&b=2
This method returns:
Content-Length: 7
Content-Type: application/x-www-form-urlencoded
When someone queries this object for the headers using
get_headers(), we'll include these. Hopefully this means that
the required headers will make it to the wire.
"""
return Headers(init_val=self.get_raw_data().get_headers())
def get_headers(self):
"""
:return: The headers which can be changed by the user during fuzzing.
:see: get_all_headers
"""
return self._headers
def get_all_headers(self):
"""
:return: Calls get_default_headers to get the default framework headers,
get_post_data_headers to get the DataContainer headers, merges that info
with the user specified headers (which live in self._headers) and
returns a Headers instance which will be sent to the wire.
"""
wire_headers = Headers()
for k, v in chain(self._headers.items(),
self.get_post_data_headers().items()):
# Ignore any keys which are already defined in the user-specified
# headers
kvalue, kreal = wire_headers.iget(k, None)
if kvalue is not None:
continue
wire_headers[k] = v
return wire_headers
def get_referer(self):
return self.get_headers().get('Referer', None)
def get_cookie(self):
return self._cookie
def get_file_vars(self):
"""
:return: A list of postdata parameters that contain a file
"""
try:
return self._post_data.get_file_vars()
except AttributeError:
return []
def test_is_variant_of_eq_keys_neq_value_types(self):
dc1 = KeyValueContainer([(u'a', ['1']), (u'b', ['c', '3'])])
dc2 = KeyValueContainer([(u'a', ['1']), (u'b', ['2', '3'])])
self.assertFalse(dc1.is_variant_of(dc2))
class FuzzableRequest(RequestMixIn, DiskItem):
"""
This class represents a fuzzable request. Fuzzable requests were created
to allow w3af plugins to be much simpler and don't really care if the
vulnerability is in the postdata, querystring, header, cookie or any other
injection point.
FuzzableRequest classes are just an easy to use representation of an HTTP
Request, which will (during the audit phase) be wrapped into a Mutant
and have its values modified.
:author: Andres Riancho ([email protected])
"""
# In most cases we don't care about these headers, even if provided by the
# user, since they will be calculated based on the attributes we are
# going to store and these won't be updated.
REMOVE_HEADERS = ('content-length', )
__slots__ = ('_method', '_cookie', '_post_data', '_headers', '_uri',
'_url', '_sent_info_comp')
def __init__(self,
uri,
method='GET',
headers=None,
cookie=None,
post_data=None):
super(FuzzableRequest, self).__init__()
# Note: Do not check for the URI/Headers type here, since I'm doing it
# in set_uri() and set_headers() already.
if cookie is not None and not isinstance(cookie, Cookie):
raise TypeError(TYPE_ERROR % ('cookie', 'Cookie'))
if post_data is not None and not isinstance(post_data, DataContainer):
raise TypeError(TYPE_ERROR % ('post_data', 'DataContainer'))
# Internal variables
self._method = method
self._cookie = Cookie() if cookie is None else cookie
self._post_data = KeyValueContainer(
) if post_data is None else post_data
# Set the headers
self._headers = None
pheaders = Headers() if headers is None else headers
self.set_headers(pheaders)
# Set the URL
self._uri = None
self._url = None
self.set_uri(uri)
# Set the internal variables
self._sent_info_comp = None
def __getstate__(self):
state = {k: getattr(self, k) for k in self.__slots__}
return state
def __setstate__(self, state):
[setattr(self, k, v) for k, v in state.iteritems()]
def get_default_headers(self):
"""
:return: The headers we want to use framework-wide for fuzzing. By
default we set the fuzzable_headers to [], which makes this
method return an empty Headers instance.
When the user sets a fuzzable_headers it will create a Headers
instance with empty values.
We then append the specific headers supplied for this
FuzzableRequest instance to the default headers. Any specific
headers override the default (empty) ones.
"""
fuzzable_headers = cf.cf.get('fuzzable_headers') or []
req_headers = [(h, '') for h in fuzzable_headers]
return Headers(init_val=req_headers)
@classmethod
def from_parts(cls, url, method='GET', post_data=None, headers=None):
"""
:return: An instance of FuzzableRequest from the provided parameters.
"""
if isinstance(url, basestring):
url = URL(url)
if post_data == '':
post_data = None
elif isinstance(post_data, basestring):
post_data = dc_from_hdrs_post(headers, post_data)
return cls(url, method=method, headers=headers, post_data=post_data)
@classmethod
def from_http_response(cls, http_response):
"""
:return: An instance of FuzzableRequest using the URL and cookie from
the http_response. The method used is "GET", and no post_data
is set.
"""
cookie = Cookie.from_http_response(http_response)
return cls(http_response.get_uri(), method='GET', cookie=cookie)
@classmethod
def from_urllib2_request(cls, request):
"""
:param request: The instance we'll use as base
:return: An instance of FuzzableRequest based on a urllib2 HTTP request
instance.
"""
headers = request.headers
headers.update(request.unredirected_hdrs)
headers = Headers(headers.items())
post_data = request.get_data() or ''
return cls.from_parts(request.url_object,
method=request.get_method(),
headers=headers,
post_data=post_data)
@classmethod
def from_form(cls, form, headers=None):
if form.get_method().upper() == 'POST':
r = cls(form.get_action(),
method=form.get_method(),
headers=headers,
post_data=form)
else:
# The default is a GET request
form_action = form.get_action()
form_action.querystring = form
r = cls(form_action, method=form.get_method(), headers=headers)
return r
def to_csv(self):
"""
Generic version of how fuzzable requests are exported:
METHOD,URL,POST_DATA
Example:
GET,http://localhost/index.php?abc=123&def=789,
POST,http://localhost/index.php,abc=123&def=789
:return: a csv str representation of the request
"""
#
# TODO: Why don't we export headers and cookies?
#
output = cStringIO.StringIO()
writer = csv.writer(output, quoting=csv.QUOTE_ALL)
writer.writerow((self.get_method(), self.get_uri(), self._post_data))
output.seek(0)
return str(output.read())[:-2]
@classmethod
def from_csv(cls, csv_line):
"""
:param csv_line: A string representing a line in a CSV file
:return: A FuzzableRequest instance
"""
row = csv.reader([csv_line], quoting=csv.QUOTE_ALL).next()
return FuzzableRequest.from_parts(row[1],
method=row[0],
post_data=row[2])
def sent(self, smth_instng):
"""
Checks if something similar to `smth_instng` was sent in the request.
This is used to remove false positives, e.g. if a grep plugin finds a
"strange" string and wants to be sure it was not generated by an audit
plugin.
This method should only be used by grep plugins which often have false
positives.
The following example shows that we sent d'z"0 but d\'z"0 will
as well be recognised as sent
TODO: This function is called MANY times, and under some circumstances
it's performance REALLY matters. We need to review this function.
:param smth_instng: The string
:return: True if something similar was sent
"""
def make_comp(heterogen_string):
"""
This basically removes characters that are hard to compare
"""
return string.translate(heterogen_string.encode('utf-8'),
TRANS_TABLE,
deletions=DELETE_CHARS)
data = self.get_data()
# This is the easy part. If it was exactly like this in the request
if data and smth_instng in data or \
smth_instng in self.get_uri() or \
smth_instng in unquote(data) or \
smth_instng in unicode(self.get_uri().url_decode()):
return True
# Ok, it's not in it but maybe something similar
# Let's set up something we can compare
if self._sent_info_comp is None:
data_encoding = self._post_data.encoding
post_data = str(self.get_data())
dec_post_data = unquote(post_data).decode(data_encoding)
data = u'%s%s%s' % (unicode(self.get_uri()), data, dec_post_data)
self._sent_info_comp = make_comp(data + unquote(data))
min_len = 3
# make the smth_instng comparable
smth_instng_comps = (make_comp(smth_instng),
make_comp(unquote(smth_instng)))
for smth_intstng_comp in smth_instng_comps:
# We don't want false negatives just because the string is
# short after making comparable
if smth_intstng_comp in self._sent_info_comp and \
len(smth_intstng_comp) >= min_len:
return True
# I didn't sent the smth_instng in any way
return False
def __hash__(self):
return hash(str(self.get_uri()) + self.get_data())
def __str__(self):
"""
:return: A string representation of this fuzzable request.
"""
short_fmt = u'Method: %s | %s'
long_fmt = u'Method: %s | %s | %s: (%s)'
if self.get_raw_data():
parameters = self.get_raw_data().get_param_names()
dc_type = self.get_raw_data().get_type()
else:
parameters = self.get_uri().querystring.get_param_names()
dc_type = self.get_uri().querystring.get_type()
if not parameters:
output = short_fmt % (self.get_method(), self.get_url())
else:
jparams = u', '.join(parameters)
output = long_fmt % (self.get_method(), self.get_url(), dc_type,
jparams)
return output.encode(DEFAULT_ENCODING)
def __unicode__(self):
return str(self).decode(encoding=DEFAULT_ENCODING, errors='ignore')
def __repr__(self):
return '<fuzzable request | %s | %s>' % (self.get_method(),
self.get_uri())
def __eq__(self, other):
"""
Two requests are equal if:
- They have the same URL
- They have the same method
- They have the same parameters
- The values for each parameter is equal
:return: True if the requests are equal.
"""
if isinstance(other, FuzzableRequest):
return (self.get_method() == other.get_method()
and self.get_uri() == other.get_uri()
and self.get_raw_data() == other.get_raw_data()
and self.get_headers() == other.get_headers())
return False
def get_eq_attrs(self):
return ['_method', '_uri', '_post_data', '_headers']
def __ne__(self, other):
return not self.__eq__(other)
def is_variant_of(self, other):
"""
Two requests are loosely equal (or variants) if:
- They have the same URL
- They have the same HTTP method
- They have the same parameter names
- The values for each parameter have the same type (int / string)
:return: True if self and other are variants.
"""
if self.get_method() != other.get_method():
return False
if self.get_url() != other.get_url():
return False
self_qs = self.get_uri().querystring
other_qs = other.get_uri().querystring
if not self_qs.is_variant_of(other_qs):
return False
return True
def set_url(self, url):
if not isinstance(url, URL):
msg = 'The "uri" parameter of a %s must be of url.URL type.'
raise TypeError(msg % type(self).__name__)
self._url = URL(url.url_string.replace(' ', '%20'))
self._uri = self._url
def set_uri(self, uri):
if not isinstance(uri, URL):
msg = 'The "uri" parameter of a %s must be of url.URL type.'
raise TypeError(msg % type(self).__name__)
self._uri = uri
self._url = uri.uri2url()
def get_querystring(self):
return self.get_uri().querystring
def set_querystring(self, new_qs):
self.get_uri().querystring = new_qs
def set_method(self, method):
self._method = method
def set_headers(self, headers):
if headers is not None and not isinstance(headers, Headers):
raise TypeError(TYPE_ERROR % ('headers', 'Headers'))
for header_name in self.REMOVE_HEADERS:
try:
headers.idel(header_name)
except KeyError:
# We don't care if they don't exist
pass
for k, v in self.get_default_headers().items():
# Ignore any keys which are already defined in the user-specified
# headers
kvalue, kreal = headers.iget(k, None)
if kvalue is not None:
continue
headers[k] = v
self._headers = headers
def set_referer(self, referer):
self._headers['Referer'] = str(referer)
def set_cookie(self, cookie):
"""
:param cookie: A Cookie object as defined in core.data.dc.cookie,
or a string.
"""
if isinstance(cookie, Cookie):
self._cookie = cookie
elif isinstance(cookie, basestring):
self._cookie = Cookie(cookie)
elif cookie is None:
self._cookie = Cookie()
else:
fmt = '[FuzzableRequest error] set_cookie received: "%s": "%s".'
error_str = fmt % (type(cookie), repr(cookie))
om.out.error(error_str)
raise BaseFrameworkException(error_str)
def get_url(self):
return self._url
def get_uri(self):
"""
:return: The URI to send in the HTTP request
"""
return self._uri
def set_data(self, post_data):
"""
Set the DataContainer which we'll use for post-data
"""
if not isinstance(post_data, DataContainer):
raise TypeError('The "post_data" parameter of a %s must be of '
'DataContainer type.' % type(self).__name__)
self._post_data = post_data
def get_data(self):
"""
The data is the string representation of the post data, in most
cases it will be used as the POSTDATA for requests.
"""
return str(self._post_data)
def get_raw_data(self):
return self._post_data
def get_method(self):
return self._method
def get_post_data_headers(self):
"""
:return: A Headers object with the headers required to send the
self._post_data to the wire. For example, if the data is
url-encoded:
a=3&b=2
This method returns:
Content-Length: 7
Content-Type: application/x-www-form-urlencoded
When someone queries this object for the headers using
get_headers(), we'll include these. Hopefully this means that
the required headers will make it to the wire.
"""
return Headers(init_val=self.get_raw_data().get_headers())
def get_headers(self):
"""
:return: The headers which can be changed by the user during fuzzing.
:see: get_all_headers
"""
return self._headers
def get_all_headers(self):
"""
:return: Calls get_default_headers to get the default framework headers,
get_post_data_headers to get the DataContainer headers, merges that info
with the user specified headers (which live in self._headers) and
returns a Headers instance which will be sent to the wire.
"""
wire_headers = Headers()
for k, v in chain(self._headers.items(),
self.get_post_data_headers().items()):
# Ignore any keys which are already defined in the user-specified
# headers
kvalue, kreal = wire_headers.iget(k, None)
if kvalue is not None:
continue
wire_headers[k] = v
return wire_headers
def get_referer(self):
return self.get_headers().get('Referer', None)
def get_cookie(self):
return self._cookie
def get_file_vars(self):
"""
:return: A list of postdata parameters that contain a file
"""
try:
return self._post_data.get_file_vars()
except AttributeError:
return []
def test_get_short_printable_repr_no_token(self):
dc = KeyValueContainer([(u'a', ['1']), (u'b', ['2', '3'])])
self.assertEqual(dc.get_short_printable_repr(), 'a=1&b=2&b=3')
