def addToReport(self, request, response=None, response_body=None, failure_string=None):
"""
Adds to the report the specified request and response.
Args:
request (dict): A dict describing the request that was made
response (instance): An instance of
:class:twisted.web.client.Response.
Note: headers is our modified True Headers version.
failure (instance): An instance of :class:twisted.internet.failure.Failure
"""
log.debug("Adding %s to report" % request)
request_headers = TrueHeaders(request['headers'])
request_response = {
'request': {
'headers': list(request_headers.getAllRawHeaders()),
'body': request['body'],
'url': request['url'],
'method': request['method']
}
}
if response:
request_response['response'] = {
'headers': list(response.headers.getAllRawHeaders()),
'body': response_body,
'code': response.code
}
if failure_string:
request_response['failure'] = failure_string
self.report['requests'].append(request_response)
def addToReport(self, request, response=None, response_body=None, failure_string=None):
"""
Adds to the report the specified request and response.
Args:
request (dict): A dict describing the request that was made
response (instance): An instance of
:class:twisted.web.client.Response.
Note: headers is our modified True Headers version.
failure (instance): An instance of :class:twisted.internet.failure.Failure
"""
log.debug("Adding %s to report" % request)
request_headers = TrueHeaders(request['headers'])
request_response = {
'request': {
'headers': list(request_headers.getAllRawHeaders()),
'body': request['body'],
'url': request['url'],
'method': request['method']
}
}
if response:
request_response['response'] = {
'headers': list(response.headers.getAllRawHeaders()),
'body': response_body,
'code': response.code
}
if failure_string:
request_response['failure'] = failure_string
self.report['requests'].append(request_response)
def check_for_tampering(self, data):
"""
Here we do checks to verify if the request we made has been tampered
with. We have 3 categories of tampering:
* **total** when the response is not a json object and therefore we were not
able to reach the ooniprobe test backend
* **request_line_capitalization** when the HTTP Request line (e.x. GET /
HTTP/1.1) does not match the capitalization we set.
* **header_field_number** when the number of headers we sent does not match
with the ones the backend received
* **header_name_capitalization** when the header field names do not match
those that we sent.
* **header_field_value** when the header field value does not match with the
one we transmitted.
"""
log.msg("Checking for tampering on %s" % self.url)
self.report['tampering'] = {
'total': False,
'request_line_capitalization': False,
'header_name_capitalization': False,
'header_field_value': False,
'header_field_number': False
}
try:
response = json.loads(data)
except ValueError:
self.report['tampering']['total'] = True
return
request_request_line = "%s / HTTP/1.1" % self.request_method
try:
response_request_line = response['request_line']
response_headers_dict = response['headers_dict']
except KeyError:
self.report['tampering']['total'] = True
return
if request_request_line != response_request_line:
self.report['tampering']['request_line_capitalization'] = True
request_headers = TrueHeaders(self.request_headers)
diff = request_headers.getDiff(TrueHeaders(response_headers_dict),
ignore=['Connection'])
if diff:
self.report['tampering']['header_field_name'] = True
else:
self.report['tampering']['header_field_name'] = False
self.report['tampering']['header_name_diff'] = list(diff)
log.msg(" total: %(total)s" % self.report['tampering'])
log.msg(" request_line_capitalization: %(request_line_capitalization)s" % self.report['tampering'])
log.msg(" header_name_capitalization: %(header_name_capitalization)s" % self.report['tampering'])
log.msg(" header_field_value: %(header_field_value)s" % self.report['tampering'])
log.msg(" header_field_number: %(header_field_number)s" % self.report['tampering'])
def test_names_not_match(self):
th = TrueHeaders(dummy_headers_dict)
self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)),
set(['Header3']))
th = TrueHeaders(dummy_headers_dict3)
self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)),
set(['Header3', 'Header4']))
def test_names_match_expect_ignore(self):
th = TrueHeaders(dummy_headers_dict)
self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2), ignore=['Header3']), set())
def test_names_not_match(self):
th = TrueHeaders(dummy_headers_dict)
self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), set(['Header3']))
th = TrueHeaders(dummy_headers_dict3)
self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), set(['Header3', 'Header4']))
def test_names_match(self):
th = TrueHeaders(dummy_headers_dict)
self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict)), set())
def doRequest(self,
url,
method="GET",
headers={},
body=None,
headers_processor=None,
body_processor=None,
use_tor=False):
"""
Perform an HTTP request with the specified method.
url: the full url path of the request
method: the HTTP Method to be used
headers: the request headers to be sent as a dict
body: the request body
headers_processor: a function to be used for processing the HTTP header
responses (defaults to self.processResponseHeaders).
This function takes as argument the HTTP headers as a
dict.
body_processory: a function to be used for processing the HTTP response
body (defaults to self.processResponseBody).
This function takes the response body as an argument.
"""
# We prefix the URL with 's' to make the connection go over the
# configured socks proxy
if use_tor:
log.debug("Using control agent for the request")
url = 's' + url
agent = self.control_agent
else:
agent = self.agent
if self.localOptions['socksproxy']:
log.debug("Using SOCKS proxy %s for request" %
(self.localOptions['socksproxy']))
url = 's' + url
log.debug("Performing request %s %s %s" % (url, method, headers))
request = {}
request['method'] = method
request['url'] = url
request['headers'] = headers
request['body'] = body
if self.randomizeUA:
log.debug("Randomizing user agent")
self.randomize_useragent(request)
log.debug("Writing to report the request")
if 'requests' not in self.report:
self.report['requests'] = []
# If we have a request body payload, set the request body to such
# content
if body:
body_producer = StringProducer(request['body'])
else:
body_producer = None
headers = TrueHeaders(request['headers'])
def errback(failure):
failure.trap(ConnectionRefusedError, SOCKSError)
if type(failure.value) is ConnectionRefusedError:
log.err("Connection refused. The backend may be down")
else:
log.err("Sock error. The SOCK proxy may be down")
self.report["failure"] = str(failure.value)
def finished(data):
return
d = agent.request(request['method'], request['url'], headers,
body_producer)
d.addErrback(errback)
d.addCallback(self._cbResponse, request, headers_processor,
body_processor)
d.addCallback(finished)
return d
def test_names_match_expect_ignore(self):
th = TrueHeaders(dummy_headers_dict)
self.assertEqual(
th.getDiff(TrueHeaders(dummy_headers_dict2), ignore=['Header3']),
set())
def test_names_match(self):
th = TrueHeaders(dummy_headers_dict)
self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict)), set())
def check_for_tampering(self, data):
"""
Here we do checks to verify if the request we made has been tampered
with. We have 3 categories of tampering:
* **total** when the response is not a json object and therefore we were not
able to reach the ooniprobe test backend
* **request_line_capitalization** when the HTTP Request line (e.x. GET /
HTTP/1.1) does not match the capitalization we set.
* **header_field_number** when the number of headers we sent does not match
with the ones the backend received
* **header_name_capitalization** when the header field names do not match
those that we sent.
* **header_field_value** when the header field value does not match with the
one we transmitted.
"""
log.msg("Checking for tampering on %s" % self.url)
self.report['tampering'] = {
'total': False,
'request_line_capitalization': False,
'header_name_capitalization': False,
'header_field_value': False,
'header_field_number': False
}
try:
response = json.loads(data)
except ValueError:
self.report['tampering']['total'] = True
return
request_request_line = "%s / HTTP/1.1" % self.request_method
try:
response_request_line = response['request_line']
response_headers_dict = response['headers_dict']
except KeyError:
self.report['tampering']['total'] = True
return
if request_request_line != response_request_line:
self.report['tampering']['request_line_capitalization'] = True
request_headers = TrueHeaders(self.request_headers)
diff = request_headers.getDiff(TrueHeaders(response_headers_dict),
ignore=['Connection'])
if diff:
self.report['tampering']['header_field_name'] = True
else:
self.report['tampering']['header_field_name'] = False
self.report['tampering']['header_name_diff'] = list(diff)
log.msg(" total: %(total)s" % self.report['tampering'])
log.msg(
" request_line_capitalization: %(request_line_capitalization)s"
% self.report['tampering'])
log.msg(
" header_name_capitalization: %(header_name_capitalization)s" %
self.report['tampering'])
log.msg(" header_field_value: %(header_field_value)s" %
self.report['tampering'])
log.msg(" header_field_number: %(header_field_number)s" %
self.report['tampering'])
def doRequest(self, url, method="GET",
headers={}, body=None, headers_processor=None,
body_processor=None, use_tor=False):
"""
Perform an HTTP request with the specified method and headers.
Args:
url (str): the full URL of the request. The scheme may be either
http, https, or httpo for http over Tor Hidden Service.
Kwargs:
method (str): the HTTP method name to use for the request
headers (dict): the request headers to send
body (str): the request body
headers_processor : a function to be used for processing the HTTP
header responses (defaults to self.processResponseHeaders).
This function takes as argument the HTTP headers as a dict.
body_processory: a function to be used for processing the HTTP
response body (defaults to self.processResponseBody). This
function takes the response body as an argument.
use_tor (bool): specify if the HTTP request should be done over Tor
or not.
"""
# We prefix the URL with 's' to make the connection go over the
# configured socks proxy
if use_tor:
log.debug("Using Tor for the request to %s" % url)
url = 's'+url
agent = self.control_agent
else:
agent = self.agent
if self.localOptions['socksproxy']:
log.debug("Using SOCKS proxy %s for request" % (self.localOptions['socksproxy']))
url = 's'+url
log.debug("Performing request %s %s %s" % (url, method, headers))
request = {}
request['method'] = method
request['url'] = url
request['headers'] = headers
request['body'] = body
if self.randomizeUA:
log.debug("Randomizing user agent")
self.randomize_useragent(request)
if 'requests' not in self.report:
self.report['requests'] = []
# If we have a request body payload, set the request body to such
# content
if body:
body_producer = StringProducer(request['body'])
else:
body_producer = None
headers = TrueHeaders(request['headers'])
def errback(failure, request):
failure_string = handleAllFailures(failure)
log.err("Error performing %s" % request)
HTTPTest.addToReport(self, request, failure_string=failure_string)
return failure
d = agent.request(request['method'], request['url'], headers,
body_producer)
d.addCallback(self._cbResponse, request, headers_processor,
body_processor)
d.addErrback(errback, request)
return d
