def addToReport(self, request, response=None, response_body=None, failure_string=None): """ Adds to the report the specified request and response. Args: request (dict): A dict describing the request that was made response (instance): An instance of :class:twisted.web.client.Response. Note: headers is our modified True Headers version. failure (instance): An instance of :class:twisted.internet.failure.Failure """ log.debug("Adding %s to report" % request) request_headers = TrueHeaders(request['headers']) request_response = { 'request': { 'headers': list(request_headers.getAllRawHeaders()), 'body': request['body'], 'url': request['url'], 'method': request['method'] } } if response: request_response['response'] = { 'headers': list(response.headers.getAllRawHeaders()), 'body': response_body, 'code': response.code } if failure_string: request_response['failure'] = failure_string self.report['requests'].append(request_response)
def addToReport(self, request, response=None, response_body=None, failure_string=None): """ Adds to the report the specified request and response. Args: request (dict): A dict describing the request that was made response (instance): An instance of :class:twisted.web.client.Response. Note: headers is our modified True Headers version. failure (instance): An instance of :class:twisted.internet.failure.Failure """ log.debug("Adding %s to report" % request) request_headers = TrueHeaders(request['headers']) request_response = { 'request': { 'headers': list(request_headers.getAllRawHeaders()), 'body': request['body'], 'url': request['url'], 'method': request['method'] } } if response: request_response['response'] = { 'headers': list(response.headers.getAllRawHeaders()), 'body': response_body, 'code': response.code } if failure_string: request_response['failure'] = failure_string self.report['requests'].append(request_response)
def check_for_tampering(self, data): """ Here we do checks to verify if the request we made has been tampered with. We have 3 categories of tampering: * **total** when the response is not a json object and therefore we were not able to reach the ooniprobe test backend * **request_line_capitalization** when the HTTP Request line (e.x. GET / HTTP/1.1) does not match the capitalization we set. * **header_field_number** when the number of headers we sent does not match with the ones the backend received * **header_name_capitalization** when the header field names do not match those that we sent. * **header_field_value** when the header field value does not match with the one we transmitted. """ log.msg("Checking for tampering on %s" % self.url) self.report['tampering'] = { 'total': False, 'request_line_capitalization': False, 'header_name_capitalization': False, 'header_field_value': False, 'header_field_number': False } try: response = json.loads(data) except ValueError: self.report['tampering']['total'] = True return request_request_line = "%s / HTTP/1.1" % self.request_method try: response_request_line = response['request_line'] response_headers_dict = response['headers_dict'] except KeyError: self.report['tampering']['total'] = True return if request_request_line != response_request_line: self.report['tampering']['request_line_capitalization'] = True request_headers = TrueHeaders(self.request_headers) diff = request_headers.getDiff(TrueHeaders(response_headers_dict), ignore=['Connection']) if diff: self.report['tampering']['header_field_name'] = True else: self.report['tampering']['header_field_name'] = False self.report['tampering']['header_name_diff'] = list(diff) log.msg(" total: %(total)s" % self.report['tampering']) log.msg(" request_line_capitalization: %(request_line_capitalization)s" % self.report['tampering']) log.msg(" header_name_capitalization: %(header_name_capitalization)s" % self.report['tampering']) log.msg(" header_field_value: %(header_field_value)s" % self.report['tampering']) log.msg(" header_field_number: %(header_field_number)s" % self.report['tampering'])
def test_names_not_match(self): th = TrueHeaders(dummy_headers_dict) self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), set(['Header3'])) th = TrueHeaders(dummy_headers_dict3) self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), set(['Header3', 'Header4']))
def test_names_match_expect_ignore(self): th = TrueHeaders(dummy_headers_dict) self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2), ignore=['Header3']), set())
def test_names_not_match(self): th = TrueHeaders(dummy_headers_dict) self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), set(['Header3'])) th = TrueHeaders(dummy_headers_dict3) self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), set(['Header3', 'Header4']))
def test_names_match(self): th = TrueHeaders(dummy_headers_dict) self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict)), set())
def doRequest(self, url, method="GET", headers={}, body=None, headers_processor=None, body_processor=None, use_tor=False): """ Perform an HTTP request with the specified method. url: the full url path of the request method: the HTTP Method to be used headers: the request headers to be sent as a dict body: the request body headers_processor: a function to be used for processing the HTTP header responses (defaults to self.processResponseHeaders). This function takes as argument the HTTP headers as a dict. body_processory: a function to be used for processing the HTTP response body (defaults to self.processResponseBody). This function takes the response body as an argument. """ # We prefix the URL with 's' to make the connection go over the # configured socks proxy if use_tor: log.debug("Using control agent for the request") url = 's' + url agent = self.control_agent else: agent = self.agent if self.localOptions['socksproxy']: log.debug("Using SOCKS proxy %s for request" % (self.localOptions['socksproxy'])) url = 's' + url log.debug("Performing request %s %s %s" % (url, method, headers)) request = {} request['method'] = method request['url'] = url request['headers'] = headers request['body'] = body if self.randomizeUA: log.debug("Randomizing user agent") self.randomize_useragent(request) log.debug("Writing to report the request") if 'requests' not in self.report: self.report['requests'] = [] # If we have a request body payload, set the request body to such # content if body: body_producer = StringProducer(request['body']) else: body_producer = None headers = TrueHeaders(request['headers']) def errback(failure): failure.trap(ConnectionRefusedError, SOCKSError) if type(failure.value) is ConnectionRefusedError: log.err("Connection refused. The backend may be down") else: log.err("Sock error. The SOCK proxy may be down") self.report["failure"] = str(failure.value) def finished(data): return d = agent.request(request['method'], request['url'], headers, body_producer) d.addErrback(errback) d.addCallback(self._cbResponse, request, headers_processor, body_processor) d.addCallback(finished) return d
def test_names_match_expect_ignore(self): th = TrueHeaders(dummy_headers_dict) self.assertEqual( th.getDiff(TrueHeaders(dummy_headers_dict2), ignore=['Header3']), set())
def test_names_match(self): th = TrueHeaders(dummy_headers_dict) self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict)), set())
def check_for_tampering(self, data): """ Here we do checks to verify if the request we made has been tampered with. We have 3 categories of tampering: * **total** when the response is not a json object and therefore we were not able to reach the ooniprobe test backend * **request_line_capitalization** when the HTTP Request line (e.x. GET / HTTP/1.1) does not match the capitalization we set. * **header_field_number** when the number of headers we sent does not match with the ones the backend received * **header_name_capitalization** when the header field names do not match those that we sent. * **header_field_value** when the header field value does not match with the one we transmitted. """ log.msg("Checking for tampering on %s" % self.url) self.report['tampering'] = { 'total': False, 'request_line_capitalization': False, 'header_name_capitalization': False, 'header_field_value': False, 'header_field_number': False } try: response = json.loads(data) except ValueError: self.report['tampering']['total'] = True return request_request_line = "%s / HTTP/1.1" % self.request_method try: response_request_line = response['request_line'] response_headers_dict = response['headers_dict'] except KeyError: self.report['tampering']['total'] = True return if request_request_line != response_request_line: self.report['tampering']['request_line_capitalization'] = True request_headers = TrueHeaders(self.request_headers) diff = request_headers.getDiff(TrueHeaders(response_headers_dict), ignore=['Connection']) if diff: self.report['tampering']['header_field_name'] = True else: self.report['tampering']['header_field_name'] = False self.report['tampering']['header_name_diff'] = list(diff) log.msg(" total: %(total)s" % self.report['tampering']) log.msg( " request_line_capitalization: %(request_line_capitalization)s" % self.report['tampering']) log.msg( " header_name_capitalization: %(header_name_capitalization)s" % self.report['tampering']) log.msg(" header_field_value: %(header_field_value)s" % self.report['tampering']) log.msg(" header_field_number: %(header_field_number)s" % self.report['tampering'])
def doRequest(self, url, method="GET", headers={}, body=None, headers_processor=None, body_processor=None, use_tor=False): """ Perform an HTTP request with the specified method and headers. Args: url (str): the full URL of the request. The scheme may be either http, https, or httpo for http over Tor Hidden Service. Kwargs: method (str): the HTTP method name to use for the request headers (dict): the request headers to send body (str): the request body headers_processor : a function to be used for processing the HTTP header responses (defaults to self.processResponseHeaders). This function takes as argument the HTTP headers as a dict. body_processory: a function to be used for processing the HTTP response body (defaults to self.processResponseBody). This function takes the response body as an argument. use_tor (bool): specify if the HTTP request should be done over Tor or not. """ # We prefix the URL with 's' to make the connection go over the # configured socks proxy if use_tor: log.debug("Using Tor for the request to %s" % url) url = 's'+url agent = self.control_agent else: agent = self.agent if self.localOptions['socksproxy']: log.debug("Using SOCKS proxy %s for request" % (self.localOptions['socksproxy'])) url = 's'+url log.debug("Performing request %s %s %s" % (url, method, headers)) request = {} request['method'] = method request['url'] = url request['headers'] = headers request['body'] = body if self.randomizeUA: log.debug("Randomizing user agent") self.randomize_useragent(request) if 'requests' not in self.report: self.report['requests'] = [] # If we have a request body payload, set the request body to such # content if body: body_producer = StringProducer(request['body']) else: body_producer = None headers = TrueHeaders(request['headers']) def errback(failure, request): failure_string = handleAllFailures(failure) log.err("Error performing %s" % request) HTTPTest.addToReport(self, request, failure_string=failure_string) return failure d = agent.request(request['method'], request['url'], headers, body_producer) d.addCallback(self._cbResponse, request, headers_processor, body_processor) d.addErrback(errback, request) return d