def test_names_not_match(self):
th = TrueHeaders(dummy_headers_dict)
self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)),
['Header3'])
th = TrueHeaders(dummy_headers_dict3)
self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)),
['Header3', 'Header4'])
def addToReport(self,
request,
response=None,
response_body=None,
failure_string=None):
"""
Adds to the report the specified request and response.
Args:
request (dict): A dict describing the request that was made
response (instance): An instance of
:class:twisted.web.client.Response.
Note: headers is our modified True Headers version.
failure (instance): An instance of :class:twisted.internet.failure.Failure
"""
log.debug("Adding %s to report" % request)
request_headers = TrueHeaders(request['headers'])
session = {
'request': {
'headers': _representHeaders(request_headers),
'body': request['body'],
'url': request['url'],
'method': request['method'],
'tor': request['tor']
},
'response': None
}
if response:
if self.localOptions.get('withoutbody', 0) is 0:
response_body = representBody(response_body)
else:
response_body = ''
# Attempt to redact the IP address of the probe from the responses
if (config.privacy.includeip is False
and config.probe_ip.address is not None
and (isinstance(response_body, str)
or isinstance(response_body, unicode))):
response_body = response_body.replace(config.probe_ip.address,
"[REDACTED]")
if (getattr(response, 'request', None)
and getattr(response.request, 'absoluteURI', None)):
session['request']['url'] = response.request.absoluteURI
session['response'] = {
'headers': _representHeaders(response.headers),
'body': response_body,
'code': response.code
}
session['failure'] = None
if failure_string:
session['failure'] = failure_string
self.report['requests'].append(session)
if response and response.previousResponse:
self.addToReport(request,
response.previousResponse,
response_body=None,
failure_string=None)
def test_redirect_works(self):
if not is_internet_connected():
raise unittest.SkipTest("Internet connection missing")
agent = FixedRedirectAgent(TrueHeadersAgent(reactor))
headers = TrueHeaders({"Spam": ["ham"]})
url = "http://httpbin.org/absolute-redirect/3"
response = yield agent.request('GET', url, headers)
body = yield readBody(response)
j = json.loads(body)
self.assertEqual(j['headers']['Spam'], 'ham')
def doRequest(self,
url,
method="GET",
headers={},
body=None,
headers_processor=None,
body_processor=None,
use_tor=False):
"""
Perform an HTTP request with the specified method and headers.
Args:
url (str): the full URL of the request. The scheme may be either
http, https, or httpo for http over Tor Hidden Service.
Kwargs:
method (str): the HTTP method name to use for the request
headers (dict): the request headers to send
body (str): the request body
headers_processor : a function to be used for processing the HTTP
header responses (defaults to self.processResponseHeaders).
This function takes as argument the HTTP headers as a dict.
body_processory: a function to be used for processing the HTTP
response body (defaults to self.processResponseBody). This
function takes the response body as an argument.
use_tor (bool): specify if the HTTP request should be done over Tor
or not.
"""
# We prefix the URL with 's' to make the connection go over the
# configured socks proxy
if use_tor:
log.debug("Using Tor for the request to %s" % url)
agent = self.control_agent
else:
agent = self.agent
if self.localOptions['socksproxy']:
log.debug("Using SOCKS proxy %s for request" %
(self.localOptions['socksproxy']))
log.debug("Performing request %s %s %s" % (url, method, headers))
request = {}
request['method'] = method
request['url'] = url
request['headers'] = headers
request['body'] = body
request['tor'] = {'exit_ip': None, 'exit_name': None}
if use_tor:
request['tor']['is_tor'] = True
else:
request['tor']['is_tor'] = False
if self.randomizeUA:
log.debug("Randomizing user agent")
self.randomize_useragent(request)
self.report['requests'] = self.report.get('requests', [])
# If we have a request body payload, set the request body to such
# content
if body:
body_producer = StringProducer(request['body'])
else:
body_producer = None
headers = TrueHeaders(request['headers'])
def errback(failure, request):
if request['tor']['is_tor']:
log.err("Error performing torified HTTP request: %s" %
request['url'])
else:
log.err("Error performing HTTP request: %s" % request['url'])
failure_string = handleAllFailures(failure)
previous_response = None
if getattr(failure, "previousResponse", None):
previous_response = failure.previousResponse
if getattr(failure, "requestLocation", None):
request['url'] = failure.requestLocation
self.addToReport(request,
failure_string=failure_string,
previous_response=previous_response)
return failure
if use_tor:
state = config.tor_state
if state:
state.add_stream_listener(StreamListener(request))
d = agent.request(request['method'], request['url'], headers,
body_producer)
d.addErrback(errback, request)
d.addCallback(self._cbResponse, request, headers_processor,
body_processor)
return d
def test_names_match_expect_ignore(self):
th = TrueHeaders(dummy_headers_dict)
self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2), ignore=['Header3']), [])
def test_names_not_match(self):
th = TrueHeaders(dummy_headers_dict)
self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), ['Header3'])
th = TrueHeaders(dummy_headers_dict3)
self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), ['Header3', 'Header4'])
def test_names_match(self):
th = TrueHeaders(dummy_headers_dict)
self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict)), [])
def check_for_tampering(self, data):
"""
Here we do checks to verify if the request we made has been tampered
with. We have 3 categories of tampering:
* **total** when the response is not a json object and therefore we were not
able to reach the ooniprobe test backend
* **request_line_capitalization** when the HTTP Request line (e.x. GET /
HTTP/1.1) does not match the capitalization we set.
* **header_field_number** when the number of headers we sent does not match
with the ones the backend received
* **header_name_capitalization** when the header field names do not match
those that we sent.
* **header_field_value** when the header field value does not match with the
one we transmitted.
"""
log.msg("Checking for tampering on %s" % self.url)
self.report['tampering'] = {
'total': False,
'request_line_capitalization': False,
'header_name_capitalization': False,
'header_field_value': False,
'header_field_number': False
}
try:
response = json.loads(data)
except ValueError:
self.report['tampering']['total'] = True
return
request_request_line = "%s / HTTP/1.1" % self.request_method
try:
response_request_line = response['request_line']
response_headers_dict = response['headers_dict']
except KeyError:
self.report['tampering']['total'] = True
return
if request_request_line != response_request_line:
self.report['tampering']['request_line_capitalization'] = True
request_headers = TrueHeaders(self.request_headers)
diff = request_headers.getDiff(TrueHeaders(response_headers_dict),
ignore=['Connection'])
if diff:
self.report['tampering']['header_field_name'] = True
else:
self.report['tampering']['header_field_name'] = False
self.report['tampering']['header_name_diff'] = list(diff)
log.msg(" total: %(total)s" % self.report['tampering'])
log.msg(
" request_line_capitalization: %(request_line_capitalization)s" %
self.report['tampering'])
log.msg(
" header_name_capitalization: %(header_name_capitalization)s" %
self.report['tampering'])
log.msg(
" header_field_value: %(header_field_value)s" %
self.report['tampering'])
log.msg(
" header_field_number: %(header_field_number)s" %
self.report['tampering'])
def check_for_tampering(self, data):
"""
Here we do checks to verify if the request we made has been tampered
with. We have 3 categories of tampering:
* **total** when the response is not a json object and therefore we were not
able to reach the ooniprobe test backend
* **request_line_capitalization** when the HTTP Request line (e.x. GET /
HTTP/1.1) does not match the capitalization we set.
* **header_field_number** when the number of headers we sent does not match
with the ones the backend received
* **header_name_capitalization** when the header field names do not match
those that we sent.
* **header_field_value** when the header field value does not match with the
one we transmitted.
"""
log.msg("Checking for tampering on %s" % self.url)
self.report['tampering'] = {
'total': False,
'request_line_capitalization': False,
'header_name_capitalization': False,
'header_field_value': False,
'header_field_number': False
}
try:
response = json.loads(data)
except ValueError:
self.report['tampering']['total'] = True
return
request_request_line = "%s / HTTP/1.1" % self.request_method
try:
response_request_line = response['request_line']
response_headers_dict = response['headers_dict']
except KeyError:
self.report['tampering']['total'] = True
return
if request_request_line != response_request_line:
self.report['tampering']['request_line_capitalization'] = True
request_headers = TrueHeaders(self.request_headers)
diff = request_headers.getDiff(TrueHeaders(response_headers_dict),
ignore=['Connection'])
if diff:
self.report['tampering']['header_field_name'] = True
else:
self.report['tampering']['header_field_name'] = False
self.report['tampering']['header_name_diff'] = list(diff)
log.msg(" total: %(total)s" % self.report['tampering'])
log.msg(
" request_line_capitalization: %(request_line_capitalization)s"
% self.report['tampering'])
log.msg(
" header_name_capitalization: %(header_name_capitalization)s" %
self.report['tampering'])
log.msg(" header_field_value: %(header_field_value)s" %
self.report['tampering'])
log.msg(" header_field_number: %(header_field_number)s" %
self.report['tampering'])
def test_names_match_expect_ignore(self):
th = TrueHeaders(dummy_headers_dict)
self.assertEqual(
th.getDiff(TrueHeaders(dummy_headers_dict2), ignore=['Header3']),
[])
def test_names_match(self):
th = TrueHeaders(dummy_headers_dict)
self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict)), [])
