def check_for_tampering(self, data):
"""
Here we do checks to verify if the request we made has been tampered
with. We have 3 categories of tampering:
* **total** when the response is not a json object and therefore we were not
able to reach the ooniprobe test backend
* **request_line_capitalization** when the HTTP Request line (e.x. GET /
HTTP/1.1) does not match the capitalization we set.
* **header_field_number** when the number of headers we sent does not match
with the ones the backend received
* **header_name_capitalization** when the header field names do not match
those that we sent.
* **header_field_value** when the header field value does not match with the
one we transmitted.
"""
log.msg("Checking for tampering on %s" % self.url)
self.report['tampering'] = {
'total': False,
'request_line_capitalization': False,
'header_name_capitalization': False,
'header_field_value': False,
'header_field_number': False
}
try:
response = json.loads(data)
except ValueError:
self.report['tampering']['total'] = True
return
request_request_line = "%s / HTTP/1.1" % self.request_method
try:
response_request_line = response['request_line']
response_headers_dict = response['headers_dict']
except KeyError:
self.report['tampering']['total'] = True
return
if request_request_line != response_request_line:
self.report['tampering']['request_line_capitalization'] = True
request_headers = TrueHeaders(self.request_headers)
diff = request_headers.getDiff(TrueHeaders(response_headers_dict),
ignore=['Connection'])
if diff:
self.report['tampering']['header_field_name'] = True
else:
self.report['tampering']['header_field_name'] = False
self.report['tampering']['header_name_diff'] = list(diff)
log.msg(" total: %(total)s" % self.report['tampering'])
log.msg(" request_line_capitalization: %(request_line_capitalization)s" % self.report['tampering'])
log.msg(" header_name_capitalization: %(header_name_capitalization)s" % self.report['tampering'])
log.msg(" header_field_value: %(header_field_value)s" % self.report['tampering'])
log.msg(" header_field_number: %(header_field_number)s" % self.report['tampering'])
def addToReport(self, request, response=None, response_body=None, failure_string=None):
"""
Adds to the report the specified request and response.
Args:
request (dict): A dict describing the request that was made
response (instance): An instance of
:class:twisted.web.client.Response.
Note: headers is our modified True Headers version.
failure (instance): An instance of :class:twisted.internet.failure.Failure
"""
log.debug("Adding %s to report" % request)
request_headers = TrueHeaders(request['headers'])
request_response = {
'request': {
'headers': list(request_headers.getAllRawHeaders()),
'body': request['body'],
'url': request['url'],
'method': request['method']
}
}
if response:
request_response['response'] = {
'headers': list(response.headers.getAllRawHeaders()),
'body': response_body,
'code': response.code
}
if failure_string:
request_response['failure'] = failure_string
self.report['requests'].append(request_response)
def test_order_preserved(self):
th = TrueHeaders()
th.setRawHeaders("HeaderFIRST", ["Value1", "Value2"])
th.setRawHeaders("headersecond", ["ValueA", "ValueB"])
th.setRawHeaders("HeaderNext", ["ValueZ", "ValueY", "ValueX"])
th.setRawHeaders("HeaderLast", ["Value2", "Value1"])
self.assertEqual(list(th.getAllRawHeaders()),[
("HeaderFIRST", ["Value1", "Value2"]),
("headersecond", ["ValueA", "ValueB"]),
("HeaderNext", ["ValueZ", "ValueY", "ValueX"]),
("HeaderLast", ["Value2", "Value1"])
])
def test_names_not_match(self):
th = TrueHeaders(dummy_headers_dict)
self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)),
['Header3'])
th = TrueHeaders(dummy_headers_dict3)
self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)),
['Header3', 'Header4'])
def addToReport(self,
request,
response=None,
response_body=None,
failure_string=None):
"""
Adds to the report the specified request and response.
Args:
request (dict): A dict describing the request that was made
response (instance): An instance of
:class:twisted.web.client.Response.
Note: headers is our modified True Headers version.
failure (instance): An instance of :class:twisted.internet.failure.Failure
"""
def _representHeaders(headers):
represented_headers = {}
for name, value in headers.getAllRawHeaders():
represented_headers[name] = value[0]
return represented_headers
def _representBody(body):
# XXX perhaps add support for decoding gzip in the future.
try:
body = unicode(body, 'ascii')
body = body.replace('\0', '')
except UnicodeDecodeError:
try:
body = unicode(body, 'utf-8')
body = body.replace('\0', '')
except UnicodeDecodeError:
body = base64Dict(body)
log.debug("Adding %s to report" % request)
request_headers = TrueHeaders(request['headers'])
session = {
'request': {
'headers': _representHeaders(request_headers),
'body': request['body'],
'url': request['url'],
'method': request['method'],
'tor': request['tor']
}
}
if response:
if self.localOptions.get('withoutbody', 0) is 0:
response_body = _representBody(response_body)
else:
response_body = ''
session['response'] = {
'headers': _representHeaders(response.headers),
'body': response_body,
'code': response.code
}
session['failure'] = None
if failure_string:
session['failure'] = failure_string
self.report['requests'].append(session)
def addToReport(self,
request,
response=None,
response_body=None,
failure_string=None):
"""
Adds to the report the specified request and response.
Args:
request (dict): A dict describing the request that was made
response (instance): An instance of
:class:twisted.web.client.Response.
Note: headers is our modified True Headers version.
failure (instance): An instance of :class:twisted.internet.failure.Failure
"""
log.debug("Adding %s to report" % request)
request_headers = TrueHeaders(request['headers'])
request_response = {
'request': {
'headers': list(request_headers.getAllRawHeaders()),
'body': request['body'],
'url': request['url'],
'method': request['method'],
'tor': request['tor']
}
}
if response:
request_response['response'] = {
'headers': list(response.headers.getAllRawHeaders()),
'body': response_body,
'code': response.code
}
if failure_string:
request_response['failure'] = failure_string
self.report['requests'].append(request_response)
def doRequest(self,
url,
method="GET",
headers={},
body=None,
headers_processor=None,
body_processor=None,
use_tor=False):
"""
Perform an HTTP request with the specified method and headers.
Args:
url (str): the full URL of the request. The scheme may be either
http, https, or httpo for http over Tor Hidden Service.
Kwargs:
method (str): the HTTP method name to use for the request
headers (dict): the request headers to send
body (str): the request body
headers_processor : a function to be used for processing the HTTP
header responses (defaults to self.processResponseHeaders).
This function takes as argument the HTTP headers as a dict.
body_processory: a function to be used for processing the HTTP
response body (defaults to self.processResponseBody). This
function takes the response body as an argument.
use_tor (bool): specify if the HTTP request should be done over Tor
or not.
"""
# We prefix the URL with 's' to make the connection go over the
# configured socks proxy
if use_tor:
log.debug("Using Tor for the request to %s" % url)
agent = self.control_agent
else:
agent = self.agent
if self.localOptions['socksproxy']:
log.debug("Using SOCKS proxy %s for request" %
(self.localOptions['socksproxy']))
log.debug("Performing request %s %s %s" % (url, method, headers))
request = {}
request['method'] = method
request['url'] = url
request['headers'] = headers
request['body'] = body
request['tor'] = {'exit_ip': None, 'exit_name': None}
if use_tor:
request['tor']['is_tor'] = True
else:
request['tor']['is_tor'] = False
if self.randomizeUA:
log.debug("Randomizing user agent")
self.randomize_useragent(request)
self.report['requests'] = self.report.get('requests', [])
# If we have a request body payload, set the request body to such
# content
if body:
body_producer = StringProducer(request['body'])
else:
body_producer = None
headers = TrueHeaders(request['headers'])
def errback(failure, request):
if request['tor']['is_tor']:
log.err("Error performing torified HTTP request: %s" %
request['url'])
else:
log.err("Error performing HTTP request: %s" % request['url'])
failure_string = handleAllFailures(failure)
self.addToReport(request, failure_string=failure_string)
return failure
if use_tor:
state = config.tor_state
if state:
state.add_stream_listener(StreamListener(request))
d = agent.request(request['method'], request['url'], headers,
body_producer)
d.addErrback(errback, request)
d.addCallback(self._cbResponse, request, headers_processor,
body_processor)
return d
def test_names_match_expect_ignore(self):
th = TrueHeaders(dummy_headers_dict)
self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2), ignore=['Header3']), [])
def test_names_not_match(self):
th = TrueHeaders(dummy_headers_dict)
self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), ['Header3'])
th = TrueHeaders(dummy_headers_dict3)
self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), ['Header3', 'Header4'])
def test_names_match(self):
th = TrueHeaders(dummy_headers_dict)
self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict)), [])
def check_for_tampering(self, data):
"""
Here we do checks to verify if the request we made has been tampered
with. We have 3 categories of tampering:
* **total** when the response is not a json object and therefore we were not
able to reach the ooniprobe test backend
* **request_line_capitalization** when the HTTP Request line (e.x. GET /
HTTP/1.1) does not match the capitalization we set.
* **header_field_number** when the number of headers we sent does not match
with the ones the backend received
* **header_name_capitalization** when the header field names do not match
those that we sent.
* **header_field_value** when the header field value does not match with the
one we transmitted.
"""
log.msg("Checking for tampering on %s" % self.url)
self.report['tampering'] = {
'total': False,
'request_line_capitalization': False,
'header_name_capitalization': False,
'header_field_value': False,
'header_field_number': False
}
try:
response = json.loads(data)
except ValueError:
self.report['tampering']['total'] = True
return
request_request_line = "%s / HTTP/1.1" % self.request_method
try:
response_request_line = response['request_line']
response_headers_dict = response['headers_dict']
except KeyError:
self.report['tampering']['total'] = True
return
if request_request_line != response_request_line:
self.report['tampering']['request_line_capitalization'] = True
request_headers = TrueHeaders(self.request_headers)
diff = request_headers.getDiff(TrueHeaders(response_headers_dict),
ignore=['Connection'])
if diff:
self.report['tampering']['header_field_name'] = True
else:
self.report['tampering']['header_field_name'] = False
self.report['tampering']['header_name_diff'] = list(diff)
log.msg(" total: %(total)s" % self.report['tampering'])
log.msg(
" request_line_capitalization: %(request_line_capitalization)s" %
self.report['tampering'])
log.msg(
" header_name_capitalization: %(header_name_capitalization)s" %
self.report['tampering'])
log.msg(
" header_field_value: %(header_field_value)s" %
self.report['tampering'])
log.msg(
" header_field_number: %(header_field_number)s" %
self.report['tampering'])
def test_names_match_expect_ignore(self):
th = TrueHeaders(dummy_headers_dict)
self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2), ignore=['Header3']), set())
def test_names_match(self):
th = TrueHeaders(dummy_headers_dict)
self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict)), set())
