def parse_qs(qstr, ignore_exc=True, encoding=DEFAULT_ENCODING):
"""
Parse a url encoded string (a=b&c=d) into a QueryString object.
:param qstr: The string to parse
:return: A QueryString object (a dict wrapper).
"""
if not isinstance(qstr, basestring):
raise TypeError("parse_qs requires a basestring as input.")
qs = QueryString(encoding=encoding)
if qstr:
# convert to string if unicode
if isinstance(qstr, unicode):
qstr = qstr.encode(encoding, "ignore")
try:
odict = OrderedDict()
for name, value in parse_qsl(qstr, keep_blank_values=True, strict_parsing=False):
if name in odict:
odict[name].append(value)
else:
odict[name] = [value]
except Exception:
if not ignore_exc:
raise BaseFrameworkException('Error while parsing "%r"' % qstr)
else:
def decode(item):
return (item[0].decode(encoding, "ignore"), [e.decode(encoding, "ignore") for e in item[1]])
qs.update((decode(item) for item in odict.items()))
return qs
def test_str_with_equal(self):
t1 = str(QueryString([('a', ['>']), ('b', ['a==1 && z >= 2', '3>2'])]))
e1 = 'a=%3E&b=a%3D%3D1%20%26%26%20z%20%3E%3D%202&b=3%3E2'
self.assertEqual(t1, e1)
t2 = str(QueryString([('a', ['x=/etc/passwd'])]))
e2 = 'a=x%3D%2Fetc%2Fpasswd'
self.assertEqual(t2, e2)
def test_merge_two_qs(self):
qs_1 = QueryString([('a', ['1'])])
qs_2 = QueryString([('b', ['2'])])
for key, values in qs_2.iteritems():
qs_1[key] = values
self.assertEqual(qs_1['b'], ['2'])
self.assertEqual(qs_1['a'], ['1'])
def test_copy_with_token(self):
dc = QueryString([('a', ['1'])])
dc.set_token(('a', 0))
dc_copy = copy.deepcopy(dc)
self.assertEqual(dc.get_token(), dc_copy.get_token())
self.assertIsNotNone(dc.get_token())
self.assertIsNotNone(dc_copy.get_token())
self.assertEqual(dc_copy.get_token().get_name(), 'a')
def test_pickle(self):
dc = QueryString([('a', ['1'])])
dc.set_token(('a', 0))
pickled_qs = cPickle.dumps(dc)
unpickled_qs = cPickle.loads(pickled_qs)
self.assertEqual(dc, unpickled_qs)
self.assertEqual(dc.keys(), unpickled_qs.keys())
self.assertEqual(dc.keys(), ['a'])
self.assertEqual(dc.get_token().get_name(), 'a')
def test_append_uniq_var_not_uniq(self):
i1 = MockInfo()
i1.set_uri(URL('http://moth/abc.html?id=1'))
i1.set_dc(QueryString([('id', '1')]))
i1.set_var('id')
i2 = MockInfo()
i2.set_uri(URL('http://moth/def.html?id=3'))
i2.set_dc(QueryString([('id', '3')]))
i2.set_var('id')
kb.append_uniq('a', 'b', i1)
kb.append_uniq('a', 'b', i2)
self.assertEqual(kb.get('a', 'b'), [i1, i2])
def __init__(self):
CrawlPlugin.__init__(self)
# Internal variables
self._first_run = True
self._already_analyzed = DiskSet(table_prefix='open_api')
# User configured variables
self._query_string_auth = QueryString()
self._header_auth = Headers()
self._no_spec_validation = False
self._custom_spec_location = ''
self._discover_fuzzable_headers = True
self._discover_fuzzable_url_parts = True
def test_append_uniq_var_not_uniq_diff_token_name(self):
i1 = MockInfo()
i1.set_uri(URL('http://moth/abc.html?id=1&foo=bar'))
i1.set_dc(QueryString([('id', ['1']), ('foo', ['bar'])]))
i1.set_token(('id', 0))
i2 = MockInfo()
i2.set_uri(URL('http://moth/abc.html?id=1&foo=bar'))
i2.set_dc(QueryString([('id', ['3']), ('foo', ['bar'])]))
i2.set_token(('foo', 0))
kb.append_uniq('a', 'b', i1)
kb.append_uniq('a', 'b', i2)
self.assertEqual(kb.get('a', 'b'), [i1, i2])
def test_append_uniq_var_default(self):
i1 = MockInfo()
i1.set_uri(URL('http://moth/abc.html?id=1'))
i1.set_dc(QueryString([('id', ['1'])]))
i1.set_token(('id', 0))
i2 = MockInfo()
i2.set_uri(URL('http://moth/abc.html?id=3'))
i2.set_dc(QueryString([('id', ['3'])]))
i2.set_token(('id', 0))
kb.append_uniq('a', 'b', i1)
kb.append_uniq('a', 'b', i2)
self.assertEqual(kb.get('a', 'b'), [i1, ])
def test_append_uniq_url_different(self):
i1 = MockInfo()
i1.set_uri(URL('http://moth/abc.html?id=1'))
i1.set_dc(QueryString([('id', ['1'])]))
i1.set_token(('id', 0))
i2 = MockInfo()
i2.set_uri(URL('http://moth/def.html?id=3'))
i2.set_dc(QueryString([('id', ['3'])]))
i2.set_token(('id', 0))
kb.append_uniq('a', 'b', i1, filter_by='URL')
kb.append_uniq('a', 'b', i2, filter_by='URL')
self.assertEqual(kb.get('a', 'b'), [i1, i2])
def test_get_query_string(self):
self.assertEqual(URL(u'http://w3af.com/a/').querystring,
QueryString({}.items()))
self.assertEqual(URL(u'http://w3af.com/foo/bar.txt?id=3').querystring,
QueryString({u'id': [u'3']}.items()))
self.assertEqual(URL(u'http://w3af.com/foo/bar.txt?id=3&id=4').querystring,
QueryString({u'id': [u'3', u'4']}.items()))
url = URL(u'http://w3af.com/foo/bar.txt?id=3&ff=4&id=5')
self.assertEqual(url.querystring,
QueryString({u'id': [u'3', u'5'], u'ff': [u'4']}.items()))
self.assertEqual(url.querystring, parse_qs(str(url.querystring)))
def test_mutant_creation(self):
qs = QueryString(self.SIMPLE_KV)
freq = FuzzableRequest(self.url)
freq.set_querystring(qs)
created_mutants = FakeMutant.create_mutants(freq, self.payloads, [],
False, self.fuzzer_config)
expected_dcs = ['a=abc&b=2', 'a=1&b=abc',
'a=def&b=2', 'a=1&b=def']
created_dcs = [str(i.get_dc()) for i in created_mutants]
self.assertEquals(expected_dcs, created_dcs)
token_0 = created_mutants[0].get_token()
self.assertIsInstance(token_0, DataToken)
self.assertEqual(token_0.get_name(), 'a')
self.assertEqual(token_0.get_original_value(), '1')
self.assertEqual(token_0.get_value(), 'abc')
token_2 = created_mutants[1].get_token()
self.assertIsInstance(token_0, DataToken)
self.assertEqual(token_2.get_name(), 'b')
self.assertEqual(token_2.get_original_value(), '2')
self.assertEqual(token_2.get_value(), 'abc')
self.assertTrue(all(isinstance(m, Mutant) for m in created_mutants))
self.assertTrue(all(m.get_mutant_class() == 'FakeMutant' for m in created_mutants))
def test_mutant_creation_repeated_params(self):
qs = QueryString([('a', ['1', '2']), ('b', ['3'])])
freq = FuzzableRequest(self.url)
freq.set_querystring(qs)
created_mutants = FakeMutant.create_mutants(freq, self.payloads, [],
False, self.fuzzer_config)
expected_dcs = ['a=abc&a=2&b=3',
'a=1&a=abc&b=3',
'a=1&a=2&b=abc',
'a=def&a=2&b=3',
'a=1&a=def&b=3',
'a=1&a=2&b=def']
created_dcs = [str(i.get_dc()) for i in created_mutants]
self.assertEquals(expected_dcs, created_dcs)
token_0 = created_mutants[0].get_token()
self.assertIsInstance(token_0, DataToken)
self.assertEqual(token_0.get_name(), 'a')
self.assertEqual(token_0.get_original_value(), '1')
self.assertEqual(token_0.get_value(), 'abc')
token_1 = created_mutants[1].get_token()
self.assertIsInstance(token_1, DataToken)
self.assertEqual(token_1.get_name(), 'a')
self.assertEqual(token_1.get_original_value(), '2')
self.assertEqual(token_1.get_value(), 'abc')
def test_append_uniq_var_specific(self):
i1 = MockInfo()
i1.set_uri(URL('http://moth/abc.html?id=1'))
i1.set_dc(QueryString([('id', '1')]))
i1.set_var('id')
i2 = MockInfo()
i2.set_uri(URL('http://moth/abc.html?id=3'))
i2.set_dc(QueryString([('id', '3')]))
i2.set_var('id')
kb.append_uniq('a', 'b', i1, filter_by='VAR')
kb.append_uniq('a', 'b', i2, filter_by='VAR')
self.assertEqual(kb.get('a', 'b'), [
i1,
])
def test_setitem_list(self):
qs = QueryString([('a', ['1'])])
qs['foo'] = ['bar']
self.assertEqual(str(qs), 'a=1&foo=bar')
qs['foo'] = ['bar', 'spam']
self.assertEqual(str(qs), 'a=1&foo=bar&foo=spam')
def test_import_csv_line_query_string(self):
irp = import_results()
qsr = irp._obj_from_csv(('GET', 'http://www.w3af.com/?id=1', ''))
self.assertIsInstance(qsr, FuzzableRequest)
self.assertEqual(qsr.get_url().get_domain(), 'www.w3af.com')
self.assertEqual(qsr.get_url().get_path(), '/')
self.assertEqual(qsr.get_method(), 'GET')
self.assertEqual(qsr.get_uri().get_querystring(),
QueryString([(u'id', [u'1'])]))
self.assertEqual(qsr.get_data(), '')
def test_mutant_creation_empty_dc(self):
qs = QueryString()
freq = FuzzableRequest(self.url)
freq.set_querystring(qs)
created_mutants = FakeMutant.create_mutants(freq, self.payloads, [],
False, self.fuzzer_config)
expected_dc_lst = []
created_dc_lst = [i.get_dc() for i in created_mutants]
self.assertEqual(created_dc_lst, expected_dc_lst)
def test_mutant_creation_ignore_params(self):
qs = QueryString(self.SIMPLE_KV)
freq = FuzzableRequest(self.url)
freq.set_querystring(qs)
created_mutants = FakeMutant.create_mutants(freq, self.payloads, ['a'],
False, self.fuzzer_config)
expected_dcs = ['a=abc&b=2', 'a=def&b=2']
created_dcs = [str(i.get_dc()) for i in created_mutants]
self.assertEqual(expected_dcs, created_dcs)
def set_querystring(self, qs):
"""
Set the query string for this URL.
"""
if isinstance(qs, DataContainer):
self._querystr = qs
elif isinstance(qs, basestring):
self._querystr = parse_qs(qs, ignore_exc=True, encoding=self.encoding)
else:
# This might fail because of the type-check performed in QueryString
# __init__, but that's ok.
self._querystr = QueryString(qs)
def get_uri(self):
"""
Query the spec / operation and return the URI (with query string
parameters included).
"""
request_dict = self._bravado_construct_request()
url = request_dict['url']
parameters = self._get_filled_parameters()
# We only send in the body the parameters that belong there
for param_name, param_def in self.operation.params.iteritems():
if param_def.location != 'query':
parameters.pop(param_name)
# If the parameter type is an array, we only send the first item
# TODO: Handle collectionFormat from the param_spec to know if
# we should send comma separated (csv) or multiple
# parameters with the same name and different values
for param_name, param_def in self.operation.params.iteritems():
if 'type' not in param_def.param_spec:
continue
if param_def.param_spec['type'] == 'array':
parameters[param_name] = parameters[param_name][0]
if parameters:
formatted_params = [(k, [str(v)]) for k, v in parameters.items()
if v is not None]
query_string = QueryString(formatted_params)
else:
# If there are no parameters, we create an empty query string, which is
# not going to be shown in the HTTP request in any way since it is
# serialized to an empty string.
query_string = QueryString()
uri = URL(url)
uri.set_querystring(query_string)
return uri
def test_mutant_copy(self):
qs = QueryString(self.SIMPLE_KV)
freq = FuzzableRequest(self.url)
freq.set_querystring(qs)
mutant = FakeMutant(freq)
mutant.set_token(('a', 0))
mutant_copy = mutant.copy()
self.assertEqual(mutant, mutant_copy)
self.assertEqual(mutant.get_token(), mutant_copy.get_token())
self.assertIsNot(None, mutant_copy.get_token())
def test_php_serialized_objects_query_string_b64(self):
url = self.url.copy()
b64obj = base64.b64encode(SERIALIZED_PHP_OBJECTS[0])
qs = QueryString([('viewstate', [b64obj])])
url.set_querystring(qs)
request = FuzzableRequest(url)
self.plugin.grep(request, self.response)
self.assertEquals(len(kb.kb.get('serialized_object',
'serialized_object')), 1)
def parse_qs(qstr, ignore_exc=True, encoding=DEFAULT_ENCODING):
"""
Parse a url encoded string (a=b&c=d) into a QueryString object.
:param qstr: The string to parse
:return: A QueryString object (a dict wrapper).
"""
if not isinstance(qstr, basestring):
raise TypeError('parse_qs requires a basestring as input.')
qs = QueryString(encoding=encoding)
if qstr:
# convert to string if unicode
if isinstance(qstr, unicode):
qstr = qstr.encode(encoding, 'ignore')
try:
odict = OrderedDict()
for name, value in parse_qsl(qstr,
keep_blank_values=True,
strict_parsing=False):
if name in odict:
odict[name].append(value)
else:
odict[name] = [value]
except Exception:
if not ignore_exc:
raise BaseFrameworkException('Error while parsing "%r"' % qstr)
else:
def decode(item):
return (item[0].decode(encoding, 'ignore'),
[e.decode(encoding, 'ignore') for e in item[1]])
qs.update((decode(item) for item in odict.items()))
return qs
def test_mutant_creation_append(self):
qs = QueryString(self.SIMPLE_KV)
freq = FuzzableRequest(self.url)
freq.set_querystring(qs)
created_mutants = FakeMutant.create_mutants(freq, self.payloads, [],
True, self.fuzzer_config)
expected_dcs = ['a=1abc&b=2', 'a=1&b=2abc',
'a=1def&b=2', 'a=1&b=2def', ]
created_dcs = [str(i.get_dc()) for i in created_mutants]
self.assertEquals(expected_dcs, created_dcs)
def test_php_serialized_objects_query_string(self):
for i, obj in enumerate(SERIALIZED_PHP_OBJECTS):
url = self.url.copy()
qs = QueryString([(str(i), [obj])])
url.set_querystring(qs)
request = FuzzableRequest(url)
self.plugin.grep(request, self.response)
self.assertEquals(len(kb.kb.get('serialized_object',
'serialized_object')), 2)
def set_querystring(self, qs):
"""
Set the query string for this URL.
"""
if isinstance(qs, DataContainer):
self._querystr = qs
elif isinstance(qs, dict):
self._querystr = QueryString(qs.items())
elif isinstance(qs, basestring):
self._querystr = parse_qs(qs,
ignore_exc=True,
encoding=self.encoding)
else:
raise TypeError, ("Invalid type '%r'; must be DataContainer, "
"dict or string" % type(qs))
def test_from_form_POST(self):
form_params = FormParameters()
form_params.add_field_by_attr_items([("name", "username"), ("value", "abc")])
form_params.add_field_by_attr_items([("name", "address"), ("value", "")])
form_params.set_action(URL('http://example.com/?id=1'))
form_params.set_method('post')
form = dc_from_form_params(form_params)
fr = FuzzableRequest.from_form(form)
self.assertIs(fr.get_uri(), form.get_action())
self.assertIs(fr.get_raw_data(), form)
self.assertEqual(fr.get_method(), 'POST')
self.assertEqual(fr.get_uri().querystring, QueryString([('id', ['1'])]))
def test_copy_with_token(self):
dc = QueryString([('a', ['1'])])
dc.set_token(('a', 0))
dc_copy = copy.deepcopy(dc)
self.assertEqual(dc.get_token(), dc_copy.get_token())
self.assertIsNotNone(dc.get_token())
self.assertIsNotNone(dc_copy.get_token())
self.assertEqual(dc_copy.get_token().get_name(), 'a')
def test_pickle(self):
dc = QueryString([('a', ['1'])])
dc.set_token(('a', 0))
pickled_qs = cPickle.dumps(dc)
unpickled_qs = cPickle.loads(pickled_qs)
self.assertEqual(dc, unpickled_qs)
self.assertEqual(dc.keys(), unpickled_qs.keys())
self.assertEqual(dc.keys(), ['a'])
self.assertEqual(dc.get_token().get_name(), 'a')
def test_mutant_generic_methods(self):
qs = QueryString(self.SIMPLE_KV)
freq = FuzzableRequest(self.url)
freq.set_querystring(qs)
created_mutants = FakeMutant.create_mutants(freq, self.payloads, [],
False, self.fuzzer_config)
mutant = created_mutants[0]
self.assertEqual(repr(mutant),
'<mutant-generic | GET | http://moth/?a=abc&b=2 >')
self.assertNotEqual(id(mutant.copy()), id(mutant))
self.assertRaises(ValueError, mutant.get_original_response_body)
body = 'abcdef123'
mutant.set_original_response_body(body)
self.assertEqual(mutant.get_original_response_body(), body)
class open_api(CrawlPlugin):
"""
Extract REST API calls from Open API specifications.
:author: Andres Riancho ([email protected])
"""
FILENAMES = ['swagger.json', 'openapi.json', 'openapi.yaml']
DIRECTORIES = [
'/', '/api/', '/api/v2/', '/api/v1/', '/api/v2.0/', '/api/v2.1/',
'/api/v1.0/', '/api/v1.1/', '/api/2.0/', '/api/2.1/', '/api/1.0/',
'/api/1.1/'
]
def __init__(self):
CrawlPlugin.__init__(self)
# Internal variables
self._first_run = True
self._already_analyzed = DiskSet(table_prefix='open_api')
# User configured variables
self._query_string_auth = QueryString()
self._header_auth = Headers()
self._no_spec_validation = False
self._custom_spec_location = ''
self._discover_fuzzable_headers = True
self._discover_fuzzable_url_parts = True
def crawl(self, fuzzable_request, debugging_id):
"""
Try to extract all the API endpoints from various locations
if no custom location specified.
:param debugging_id: A unique identifier for this call to discover()
:param fuzzable_request: A fuzzable_request instance that contains
(among other things) the URL to test.
"""
self._enable_file_name_fuzzing()
if self._has_custom_spec_location():
self._analyze_custom_spec()
else:
self._analyze_common_paths(fuzzable_request, debugging_id)
self._analyze_current_path(fuzzable_request, debugging_id)
def _enable_file_name_fuzzing(self):
"""
Enable file name fuzzing:
http://w3af.org/api/1.0/pets/{fuzz-this-part}
Users are not going to remember to enable this in misc-settings, and
most of the APIs which are documented with Open API are REST APIs,
so it makes sense to enable this automatically here.
:return: None
"""
if self._first_run and not self._discover_fuzzable_url_parts:
cf.cf.save('fuzz_url_filenames', True)
cf.cf.save('fuzz_url_parts', True)
def _should_analyze(self, url):
"""
Makes sure that we only analyze a URL once, this reduces the number
of HTTP requests and the CPU usage required for parsing the
response
:param url: The URL we want to analyze
:return: True if we never analyzed this URL before
"""
if url in self._already_analyzed:
return False
self._already_analyzed.add(url)
return True
def _analyze_common_paths(self, fuzzable_request, debugging_id):
"""
Try to find the open api specification in the most common paths,
extract all the REST API endpoints when found.
This is run only the first time the plugin is called.
:return: None, everything we find is sent to the core.
"""
if not self._first_run:
return
self._first_run = False
args = izip(self._spec_url_generator_common(fuzzable_request),
repeat(debugging_id))
self.worker_pool.map_multi_args(self._extract_api_calls, args)
def _extract_api_calls(self, spec_url, debugging_id):
"""
HTTP GET the `spec_url` and try to parse it. Send all the newly found
fuzzable requests to the core after adding any authentication data
that might have been configured.
:return: None
"""
#
# Merge the user-configured authentication query string (if any)
# with the spec_url query string
#
qs = spec_url.get_querystring()
for key, values in self._query_string_auth.iteritems():
qs[key] = values
spec_url.set_querystring(qs)
#
# Also add the authentication headers to the request (if any)
#
# Disable the cache because we're sending auth headers which might
# confuse the cache implementation
#
http_response = self._uri_opener.GET(spec_url,
headers=self._header_auth,
cache=False,
debugging_id=debugging_id)
if is_404(http_response):
return
self._extract_api_calls_from_response(spec_url, http_response)
def _extract_api_calls_from_response(self, spec_url, http_response):
"""
Try to parse an API specification from an HTTP response.
Send all the newly found fuzzable requests to the core
after adding any authentication data that might have been configured.
:parm spec_url: A URL to API specification
:param http_response: An HTTP response
:return: None
"""
if not OpenAPI.can_parse(http_response):
return
om.out.debug('OpenAPI parser is about to parse %s' % spec_url)
parser = OpenAPI(http_response, self._no_spec_validation,
self._discover_fuzzable_headers,
self._discover_fuzzable_url_parts)
parser.parse()
self._report_to_kb_if_needed(http_response, parser)
self._send_spec_to_core(spec_url)
om.out.debug('OpenAPI parser identified %s API calls' %
len(parser.get_api_calls()))
for api_call in parser.get_api_calls():
if not self._is_target_domain(api_call):
continue
api_call = self._set_authentication_data(api_call)
self.output_queue.put(api_call)
def _send_spec_to_core(self, spec_url):
fuzzable_request = FuzzableRequest(spec_url, method='GET')
self.output_queue.put(fuzzable_request)
@staticmethod
def _is_target_domain(fuzzable_request):
"""
:param fuzzable_request: The api call as a fuzzable request
:return: True if the target domain matches
"""
targets = cf.cf.get('targets')
if not targets:
return False
target_domain = targets[0].get_domain()
api_call_domain = fuzzable_request.get_url().get_domain()
if target_domain == api_call_domain:
return True
om.out.debug('The OpenAPI specification has operations which point'
' to a domain (%s) outside the defined target (%s).'
' Ignoring the operation to prevent scanning out of scope'
' targets.' % (api_call_domain, target_domain))
return False
def _report_to_kb_if_needed(self, http_response, parser):
"""
If the parser did find something, then we report it to the KB.
:param http_response: The HTTP response that was parsed
:param parser: The OpenAPI parser instance
:return: None
"""
if not parser.get_api_calls() and parser.get_parsing_errors():
desc = (
'An Open API specification was found at: "%s", but the scanner'
' was unable to extract any API endpoints. In most cases this'
' is because of a syntax error in the Open API specification.\n'
'\n'
'Use https://editor.swagger.io/ to inspect the Open API'
' specification, identify and fix any issues and try again.\n'
'\n'
'The errors found by the parser were:\n'
'\n - %s')
desc %= (http_response.get_url(),
'\n - '.join(parser.get_parsing_errors()))
i = Info('Failed to parse Open API specification', desc,
http_response.id, self.get_name())
i.set_url(http_response.get_url())
kb.kb.append(self, 'open_api', i)
om.out.error(i.get_desc())
return
# Save it to the kb!
desc = ('An Open API specification was found at: "%s", the scanner'
' was able to extract %s API endpoints which will be audited'
' for vulnerabilities.')
desc %= (http_response.get_url(), len(parser.get_api_calls()))
i = Info('Open API specification found', desc, http_response.id,
self.get_name())
i.set_url(http_response.get_url())
kb.kb.append(self, 'open_api', i)
om.out.information(i.get_desc())
# Warn the user about missing credentials
if self._query_string_auth or self._header_auth:
return
desc = (
'An Open API specification was found at: "%s", but no credentials'
' were provided in the `open_api` plugin. The scanner will try'
' to audit the identified endpoints but coverage will most likely'
' be reduced due to missing authentication.')
desc %= http_response.get_url()
i = Info('Open API missing credentials', desc, http_response.id,
self.get_name())
i.set_url(http_response.get_url())
kb.kb.append(self, 'open_api', i)
om.out.information(i.get_desc())
def _set_authentication_data(self, fuzzable_request):
"""
:param fuzzable_request: The fuzzable request as returned by the parser
:return: The same fuzzable request as before, but adding authentication
data configured by the user, such as headers and query string
parameters.
"""
headers = fuzzable_request.get_headers()
uri = fuzzable_request.get_uri()
query_string = uri.get_querystring()
if self._header_auth:
for header_name, header_value in self._header_auth.iteritems():
headers[header_name] = header_value
if self._query_string_auth:
for qs_param, qs_value in self._query_string_auth.iteritems():
query_string[qs_param] = qs_value
uri.set_querystring(query_string)
fuzzable_request.set_uri(uri)
fuzzable_request.set_headers(headers)
return fuzzable_request
def _spec_url_generator_common(self, fuzzable_request):
"""
Generate the potential locations for the open api specification
:param fuzzable_request: The fuzzable request we get from the core
:return: URLs to test
"""
base_url = fuzzable_request.get_url().base_url()
for directory in self.DIRECTORIES:
for filename in self.FILENAMES:
spec_url = base_url.url_join('%s%s' % (directory, filename))
if not self._should_analyze(spec_url):
continue
yield spec_url
def _spec_url_generator_current_path(self, fuzzable_request):
"""
Generate the potential locations for the open api specification
based on the current path
:param fuzzable_request: The fuzzable request we get from the core
:return: URLs to test
"""
url = fuzzable_request.get_url()
# If the user set the swagger.json URL as target, we want to test it
if self._should_analyze(url):
yield url
# Now we create some URLs based on the received URL
for directory_url in url.get_directories():
for filename in self.FILENAMES:
spec_url = directory_url.url_join(filename)
if not self._should_analyze(spec_url):
continue
yield spec_url
def _analyze_current_path(self, fuzzable_request, debugging_id):
"""
Try to find the common files in the current path.
This is faster than `_analyze_common_paths` since it doesn't test all
the directories (such as /api/ , /api/v2/, etc).
:return: None, we send everything we find to the core.
"""
args = izip(self._spec_url_generator_current_path(fuzzable_request),
repeat(debugging_id))
self.worker_pool.map_multi_args(self._extract_api_calls, args)
def _has_custom_spec_location(self):
"""
Checks if the plugin is configured to use a custom API specification
from a local file.
:return: True if the plugin is configured to read a custom API spec
"""
return self._custom_spec_location != ''
def _analyze_custom_spec(self):
"""
Loads a custom API specification from a local file, and try to parse it.
:return: None
"""
if not self._first_run:
return
self._first_run = False
url = URL('file://%s' % os.path.abspath(self._custom_spec_location))
ext = os.path.splitext(self._custom_spec_location)[1][1:].lower()
if ext not in ('yaml', 'json'):
om.out.error('Skip loading custom API spec '
'because of unknown file extension: %s' % ext)
return
with open(self._custom_spec_location, 'r') as f:
custom_spec_as_string = f.read()
headers = Headers([('content-type', 'application/%s' % ext)])
http_response = HTTPResponse(200,
custom_spec_as_string,
headers,
url,
url,
_id=1)
self._extract_api_calls_from_response(url, http_response)
def get_options(self):
"""
:return: A list of option objects for this plugin.
"""
ol = OptionList()
d = 'Query string parameters to add in each API request'
h = ('Some REST APIs use query string parameters, such as `api_key`'
' for authentication. Set this parameter to configure one or more'
' query string parameters which will be added to each API HTTP'
' request. An example value for this field is: "api_key=0x12345"')
o = opt_factory('query_string_auth',
self._query_string_auth,
d,
QUERY_STRING,
help=h)
ol.add(o)
d = 'Headers to add in each API request'
h = (
'Some REST APIs use HTTP headers, such as `X-Authenticate` or `Basic`'
' for authentication. Set this parameter to configure one or more'
' HTTP headers which will be added to each API request.'
' An example value for this field is: "Basic: bearer 0x12345"')
o = opt_factory('header_auth', self._header_auth, d, HEADER, help=h)
ol.add(o)
d = 'Disable Open API spec validation'
h = 'By default, the plugin validates Open API specification before extracting endpoints.'
o = opt_factory('no_spec_validation',
self._no_spec_validation,
d,
BOOL,
help=h)
ol.add(o)
d = 'Path to Open API specification'
h = (
'By default, the plugin looks for the API specification on the target,'
' but sometimes applications do not provide an API specification.'
' Set this parameter to specify a local path to the API specification.'
' The file must have .json or .yaml extension.')
o = opt_factory('custom_spec_location',
self._custom_spec_location,
d,
INPUT_FILE,
help=h)
ol.add(o)
d = 'Automatic HTTP header discovery for further testing'
h = (
'By default, the plugin looks for parameters which are passed to endpoints via HTTP headers,'
' and enables them for further testing.'
' Set this options to False if you would like to disable this feature.'
' You can also set `misc-settings.fuzzable_headers` option to test only specific headers.'
)
o = opt_factory('discover_fuzzable_headers',
self._discover_fuzzable_headers,
d,
BOOL,
help=h)
ol.add(o)
d = 'Automatic path parameter discovery for further testing'
h = (
'By default, URLs discovered by this plugin allow other plugins'
' to inject content into the path only at locations declared as path'
' parameters in the Open API specification.'
'\n'
' For example, if the Open API specification declares an endpoint with the path'
' `/store/product-{productID}`, only the `{productID}` part of the URL will be'
' modified during fuzzing.'
'\n'
' Set this option to False if you would like to disable this feature,'
' and instead fuzz all path segments. If this option is set to False,'
' the plugin will automatically set `misc-settings.fuzz_url_parts`'
' and `misc-settings.fuzz_url_filenames` to True')
o = opt_factory('discover_fuzzable_url_parts',
self._discover_fuzzable_url_parts,
d,
BOOL,
help=h)
ol.add(o)
return ol
def set_options(self, options_list):
"""
This method sets all the options that are configured using the user
interface generated by the framework using the result of get_options().
:param options_list: A dictionary with the options for the plugin.
:return: No value is returned.
"""
self._query_string_auth = options_list['query_string_auth'].get_value()
self._header_auth = options_list['header_auth'].get_value()
self._no_spec_validation = options_list[
'no_spec_validation'].get_value()
self._custom_spec_location = options_list[
'custom_spec_location'].get_value()
self._discover_fuzzable_headers = options_list[
'discover_fuzzable_headers'].get_value()
self._discover_fuzzable_url_parts = options_list[
'discover_fuzzable_url_parts'].get_value()
def get_long_desc(self):
"""
:return: A DETAILED description of the plugin functions and features.
"""
return """
def test_parse_qs_case07(self):
self.assertRaises(TypeError, parse_qs, QueryString())
def test_parse_qs_case06(self):
self.assertEqual(
parse_qs(u'%B1%D0%B1%D1=%B1%D6%B1%D7', encoding='euc-jp'),
QueryString([
(u'\u9834\u82f1', [u'\u75ab\u76ca']),
]))
def test_encoding_special_unicode(self):
qs = QueryString([('a', [u'✓'])])
qs.set_token(('a', 0))
self.assertEqual(str(qs), 'a=%E2%9C%93')
def create_fuzzable_request_from_parts(url, method='GET', post_data='',
add_headers=None):
"""
Creates a fuzzable request based on the input parameters.
:param req_url: A URL object
:param method: A string that represents the method ('GET', 'POST', etc)
:param post_data: A string that represents the postdata.
:param add_headers: A Headers object that holds the headers. If `req_url` is a
request then this dict will be merged with the request's
headers.
"""
if add_headers is not None and not isinstance(add_headers, Headers):
raise ValueError('create_fuzzable_request requires Headers object.')
if not isinstance(url, URL):
raise TypeError('Requires URL to create FuzzableRequest.')
headers = add_headers or Headers()
# Just a query string request! No postdata
if not post_data:
return HTTPQSRequest(url, method, headers)
else:
# Seems to be something that has post data
data = {}
conttype, header_name = headers.iget('content-type', '')
if conttype:
del headers[header_name]
contlen, header_name = headers.iget('content-length', '')
if contlen:
del headers[header_name]
#
# Case #1 - multipart form data - prepare data container
#
if conttype.startswith('multipart/form-data'):
pdict = cgi.parse_header(conttype)[1]
try:
dc = cgi.parse_multipart(StringIO(post_data), pdict)
except Exception, e:
msg = 'Multipart form data is invalid, exception: "%s".' \
' Returning our best match HTTPPostDataRequest.'
om.out.debug(msg % e)
empty_data = QueryString()
return HTTPPostDataRequest(url, method, headers, dc=empty_data)
else:
data = QueryString()
data.update(dc)
# Please note that the QueryString is just a container for the
# information. When the HTTPPostDataRequest is sent it should
# be serialized into multipart again by the MultipartPostHandler
# because the headers contain the multipart/form-data header
headers['content-type'] = conttype
return HTTPPostDataRequest(url, method, headers, dc=data)
#
# Case #2 - JSON request
#
try:
data = json.loads(post_data)
except:
pass
else:
if data:
return JSONPostDataRequest(url, method, headers, dc=data)
#
# Case #3 - XMLRPC request
#
if all(map(lambda stop: stop in post_data.lower(), XMLRPC_WORDS)):
return XMLRPCRequest(post_data, url, method, headers)
#
# Case #4 - a typical post request
#
try:
data = parse_qs(post_data)
except:
om.out.debug('Failed to create a data container that '
'can store this data: "' + post_data + '".')
else:
# Finally create request
return HTTPPostDataRequest(url, method, headers, dc=data)
return None
