def test_names_not_match(self): th = TrueHeaders(dummy_headers_dict) self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), ['Header3']) th = TrueHeaders(dummy_headers_dict3) self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2)), ['Header3', 'Header4'])
def addToReport(self, request, response=None, response_body=None, failure_string=None): """ Adds to the report the specified request and response. Args: request (dict): A dict describing the request that was made response (instance): An instance of :class:twisted.web.client.Response. Note: headers is our modified True Headers version. failure (instance): An instance of :class:twisted.internet.failure.Failure """ log.debug("Adding %s to report" % request) request_headers = TrueHeaders(request['headers']) session = { 'request': { 'headers': _representHeaders(request_headers), 'body': request['body'], 'url': request['url'], 'method': request['method'], 'tor': request['tor'] }, 'response': None } if response: if self.localOptions.get('withoutbody', 0) is 0: response_body = representBody(response_body) else: response_body = '' # Attempt to redact the IP address of the probe from the responses if (config.privacy.includeip is False and config.probe_ip.address is not None and (isinstance(response_body, str) or isinstance(response_body, unicode))): response_body = response_body.replace(config.probe_ip.address, "[REDACTED]") if (getattr(response, 'request', None) and getattr(response.request, 'absoluteURI', None)): session['request']['url'] = response.request.absoluteURI session['response'] = { 'headers': _representHeaders(response.headers), 'body': response_body, 'code': response.code } session['failure'] = None if failure_string: session['failure'] = failure_string self.report['requests'].append(session) if response and response.previousResponse: self.addToReport(request, response.previousResponse, response_body=None, failure_string=None)
def test_redirect_works(self): if not is_internet_connected(): raise unittest.SkipTest("Internet connection missing") agent = FixedRedirectAgent(TrueHeadersAgent(reactor)) headers = TrueHeaders({"Spam": ["ham"]}) url = "http://httpbin.org/absolute-redirect/3" response = yield agent.request('GET', url, headers) body = yield readBody(response) j = json.loads(body) self.assertEqual(j['headers']['Spam'], 'ham')
def doRequest(self, url, method="GET", headers={}, body=None, headers_processor=None, body_processor=None, use_tor=False): """ Perform an HTTP request with the specified method and headers. Args: url (str): the full URL of the request. The scheme may be either http, https, or httpo for http over Tor Hidden Service. Kwargs: method (str): the HTTP method name to use for the request headers (dict): the request headers to send body (str): the request body headers_processor : a function to be used for processing the HTTP header responses (defaults to self.processResponseHeaders). This function takes as argument the HTTP headers as a dict. body_processory: a function to be used for processing the HTTP response body (defaults to self.processResponseBody). This function takes the response body as an argument. use_tor (bool): specify if the HTTP request should be done over Tor or not. """ # We prefix the URL with 's' to make the connection go over the # configured socks proxy if use_tor: log.debug("Using Tor for the request to %s" % url) agent = self.control_agent else: agent = self.agent if self.localOptions['socksproxy']: log.debug("Using SOCKS proxy %s for request" % (self.localOptions['socksproxy'])) log.debug("Performing request %s %s %s" % (url, method, headers)) request = {} request['method'] = method request['url'] = url request['headers'] = headers request['body'] = body request['tor'] = {'exit_ip': None, 'exit_name': None} if use_tor: request['tor']['is_tor'] = True else: request['tor']['is_tor'] = False if self.randomizeUA: log.debug("Randomizing user agent") self.randomize_useragent(request) self.report['requests'] = self.report.get('requests', []) # If we have a request body payload, set the request body to such # content if body: body_producer = StringProducer(request['body']) else: body_producer = None headers = TrueHeaders(request['headers']) def errback(failure, request): if request['tor']['is_tor']: log.err("Error performing torified HTTP request: %s" % request['url']) else: log.err("Error performing HTTP request: %s" % request['url']) failure_string = handleAllFailures(failure) previous_response = None if getattr(failure, "previousResponse", None): previous_response = failure.previousResponse if getattr(failure, "requestLocation", None): request['url'] = failure.requestLocation self.addToReport(request, failure_string=failure_string, previous_response=previous_response) return failure if use_tor: state = config.tor_state if state: state.add_stream_listener(StreamListener(request)) d = agent.request(request['method'], request['url'], headers, body_producer) d.addErrback(errback, request) d.addCallback(self._cbResponse, request, headers_processor, body_processor) return d
def test_names_match_expect_ignore(self): th = TrueHeaders(dummy_headers_dict) self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict2), ignore=['Header3']), [])
def test_names_match(self): th = TrueHeaders(dummy_headers_dict) self.assertEqual(th.getDiff(TrueHeaders(dummy_headers_dict)), [])
def check_for_tampering(self, data): """ Here we do checks to verify if the request we made has been tampered with. We have 3 categories of tampering: * **total** when the response is not a json object and therefore we were not able to reach the ooniprobe test backend * **request_line_capitalization** when the HTTP Request line (e.x. GET / HTTP/1.1) does not match the capitalization we set. * **header_field_number** when the number of headers we sent does not match with the ones the backend received * **header_name_capitalization** when the header field names do not match those that we sent. * **header_field_value** when the header field value does not match with the one we transmitted. """ log.msg("Checking for tampering on %s" % self.url) self.report['tampering'] = { 'total': False, 'request_line_capitalization': False, 'header_name_capitalization': False, 'header_field_value': False, 'header_field_number': False } try: response = json.loads(data) except ValueError: self.report['tampering']['total'] = True return request_request_line = "%s / HTTP/1.1" % self.request_method try: response_request_line = response['request_line'] response_headers_dict = response['headers_dict'] except KeyError: self.report['tampering']['total'] = True return if request_request_line != response_request_line: self.report['tampering']['request_line_capitalization'] = True request_headers = TrueHeaders(self.request_headers) diff = request_headers.getDiff(TrueHeaders(response_headers_dict), ignore=['Connection']) if diff: self.report['tampering']['header_field_name'] = True else: self.report['tampering']['header_field_name'] = False self.report['tampering']['header_name_diff'] = list(diff) log.msg(" total: %(total)s" % self.report['tampering']) log.msg( " request_line_capitalization: %(request_line_capitalization)s" % self.report['tampering']) log.msg( " header_name_capitalization: %(header_name_capitalization)s" % self.report['tampering']) log.msg( " header_field_value: %(header_field_value)s" % self.report['tampering']) log.msg( " header_field_number: %(header_field_number)s" % self.report['tampering'])
def check_for_tampering(self, data): """ Here we do checks to verify if the request we made has been tampered with. We have 3 categories of tampering: * **total** when the response is not a json object and therefore we were not able to reach the ooniprobe test backend * **request_line_capitalization** when the HTTP Request line (e.x. GET / HTTP/1.1) does not match the capitalization we set. * **header_field_number** when the number of headers we sent does not match with the ones the backend received * **header_name_capitalization** when the header field names do not match those that we sent. * **header_field_value** when the header field value does not match with the one we transmitted. """ log.msg("Checking for tampering on %s" % self.url) self.report['tampering'] = { 'total': False, 'request_line_capitalization': False, 'header_name_capitalization': False, 'header_field_value': False, 'header_field_number': False } try: response = json.loads(data) except ValueError: self.report['tampering']['total'] = True return request_request_line = "%s / HTTP/1.1" % self.request_method try: response_request_line = response['request_line'] response_headers_dict = response['headers_dict'] except KeyError: self.report['tampering']['total'] = True return if request_request_line != response_request_line: self.report['tampering']['request_line_capitalization'] = True request_headers = TrueHeaders(self.request_headers) diff = request_headers.getDiff(TrueHeaders(response_headers_dict), ignore=['Connection']) if diff: self.report['tampering']['header_field_name'] = True else: self.report['tampering']['header_field_name'] = False self.report['tampering']['header_name_diff'] = list(diff) log.msg(" total: %(total)s" % self.report['tampering']) log.msg( " request_line_capitalization: %(request_line_capitalization)s" % self.report['tampering']) log.msg( " header_name_capitalization: %(header_name_capitalization)s" % self.report['tampering']) log.msg(" header_field_value: %(header_field_value)s" % self.report['tampering']) log.msg(" header_field_number: %(header_field_number)s" % self.report['tampering'])
def test_names_match_expect_ignore(self): th = TrueHeaders(dummy_headers_dict) self.assertEqual( th.getDiff(TrueHeaders(dummy_headers_dict2), ignore=['Header3']), [])