def test_copy(self): """ L{Headers.copy} creates a new independent copy of an existing L{Headers} instance, allowing future modifications without impacts between the copies. """ h = Headers() h.setRawHeaders("test\u00E1", ["foo\u2603"]) i = h.copy() # The copy contains the same value as the original self.assertEqual(i.getRawHeaders("test\u00E1"), ["foo\u2603"]) self.assertEqual(i.getRawHeaders(b"test\xe1"), [b"foo\xe2\x98\x83"]) # Add a header to the original h.addRawHeader("test\u00E1", "bar") # Verify that the copy has not changed self.assertEqual(i.getRawHeaders("test\u00E1"), ["foo\u2603"]) self.assertEqual(i.getRawHeaders(b"test\xe1"), [b"foo\xe2\x98\x83"]) # Add a header to the copy i.addRawHeader("test\u00E1", b"baz") # Verify that the orignal does not have it self.assertEqual(h.getRawHeaders("test\u00E1"), ["foo\u2603", "bar"]) self.assertEqual(h.getRawHeaders(b"test\xe1"), [b"foo\xe2\x98\x83", b"bar"])
def test_copy(self): """ L{Headers.copy} creates a new independent copy of an existing L{Headers} instance, allowing future modifications without impacts between the copies. """ h = Headers() h.setRawHeaders(u'test\u00E1', [u'foo\u2603']) i = h.copy() # The copy contains the same value as the original self.assertEqual(i.getRawHeaders(u'test\u00E1'), [u'foo\u2603']) self.assertEqual(i.getRawHeaders(b'test\xe1'), [b'foo\xe2\x98\x83']) # Add a header to the original h.addRawHeader(u'test\u00E1', u'bar') # Verify that the copy has not changed self.assertEqual(i.getRawHeaders(u'test\u00E1'), [u'foo\u2603']) self.assertEqual(i.getRawHeaders(b'test\xe1'), [b'foo\xe2\x98\x83']) # Add a header to the copy i.addRawHeader(u'test\u00E1', b'baz') # Verify that the orignal does not have it self.assertEqual( h.getRawHeaders(u'test\u00E1'), [u'foo\u2603', u'bar']) self.assertEqual( h.getRawHeaders(b'test\xe1'), [b'foo\xe2\x98\x83', b'bar'])
def test_addRawHeader(self): """ L{Headers.addRawHeader} adds a new value for a given header. """ h = Headers() h.addRawHeader("test", "lemur") self.assertEqual(h.getRawHeaders("test"), ["lemur"]) h.addRawHeader("test", "panda") self.assertEqual(h.getRawHeaders("test"), ["lemur", "panda"])
def test_addRawHeader(self): """ L{Headers.addRawHeader} adds a new value for a given header. """ h = Headers() h.addRawHeader(b"test", b"lemur") self.assertEqual(h.getRawHeaders(b"test"), [b"lemur"]) h.addRawHeader(b"test", b"panda") self.assertEqual(h.getRawHeaders(b"test"), [b"lemur", b"panda"])
def test_initializer(self): """ The header values passed to L{Headers.__init__} can be retrieved via L{Headers.getRawHeaders}. If a L{bytes} argument is given, it returns L{bytes} values, and if a L{unicode} argument is given, it returns L{unicode} values. Both are the same header value, just encoded or decoded. """ h = Headers({"Foo": ["bar"]}) self.assertEqual(h.getRawHeaders(b"foo"), [b"bar"]) self.assertEqual(h.getRawHeaders("foo"), ["bar"])
def test_getRawHeadersWithDefaultMatchingValue(self): """ If the object passed as the value list to L{Headers.setRawHeaders} is later passed as a default to L{Headers.getRawHeaders}, the result nevertheless contains decoded values. """ h = Headers() default = [b"value"] h.setRawHeaders(b"key", default) self.assertIsInstance(h.getRawHeaders("key", default)[0], str) self.assertEqual(h.getRawHeaders("key", default), ["value"])
def test_getRawHeaders(self): """ L{Headers.getRawHeaders} returns the values which have been set for a given header. """ h = Headers() h.setRawHeaders("test\u00E1", ["lemur"]) self.assertEqual(h.getRawHeaders("test\u00E1"), ["lemur"]) self.assertEqual(h.getRawHeaders("Test\u00E1"), ["lemur"]) self.assertEqual(h.getRawHeaders(b"test\xe1"), [b"lemur"]) self.assertEqual(h.getRawHeaders(b"Test\xe1"), [b"lemur"])
def test_getRawHeadersWithDefaultMatchingValue(self): """ If the object passed as the value list to L{Headers.setRawHeaders} is later passed as a default to L{Headers.getRawHeaders}, the result nevertheless contains encoded values. """ h = Headers() default = [u"value"] h.setRawHeaders(b"key", default) self.assertIsInstance(h.getRawHeaders(b"key", default)[0], bytes) self.assertEqual(h.getRawHeaders(b"key", default), [b"value"])
def test_getRawHeaders(self): """ L{Headers.getRawHeaders} returns the values which have been set for a given header. """ h = Headers() h.setRawHeaders(u"test\u00E1", [u"lemur"]) self.assertEqual(h.getRawHeaders(u"test\u00E1"), [u"lemur"]) self.assertEqual(h.getRawHeaders(u"Test\u00E1"), [u"lemur"]) self.assertEqual(h.getRawHeaders(b"test\xe1"), [b"lemur"]) self.assertEqual(h.getRawHeaders(b"Test\xe1"), [b"lemur"])
def test_getRawHeadersDefaultValue(self): """ L{Headers.getRawHeaders} returns the specified default value when no header is found. """ h = Headers() default = object() self.assertIdentical(h.getRawHeaders(u"test", default), default) self.assertIdentical(h.getRawHeaders(u"test", None), None) self.assertEqual(h.getRawHeaders(u"test", [None]), [None]) self.assertEqual( h.getRawHeaders(u"test", [u"\N{SNOWMAN}"]), [u"\N{SNOWMAN}"], )
def test_getRawHeadersDefaultValue(self): """ L{Headers.getRawHeaders} returns the specified default value when no header is found. """ h = Headers() default = object() self.assertIdentical(h.getRawHeaders("test", default), default) self.assertIdentical(h.getRawHeaders("test", None), None) self.assertEqual(h.getRawHeaders("test", [None]), [None]) self.assertEqual( h.getRawHeaders("test", ["\N{SNOWMAN}"]), ["\N{SNOWMAN}"], )
def test_setRawHeaders(self): """ L{Headers.setRawHeaders} sets the header values for the given header name to the sequence of strings, encoded. """ rawValue = [u"value1", u"value2"] rawEncodedValue = [b"value1", b"value2"] h = Headers() h.setRawHeaders("test", rawValue) self.assertTrue(h.hasHeader(b"test")) self.assertTrue(h.hasHeader(b"Test")) self.assertTrue(h.hasHeader("test")) self.assertTrue(h.hasHeader("Test")) self.assertEqual(h.getRawHeaders("test"), rawValue) self.assertEqual(h.getRawHeaders(b"test"), rawEncodedValue)
def test_setRawHeaders(self): """ L{Headers.setRawHeaders} sets the header values for the given header name to the sequence of strings, encoded. """ rawValue = ["value1", "value2"] rawEncodedValue = [b"value1", b"value2"] h = Headers() h.setRawHeaders("test", rawValue) self.assertTrue(h.hasHeader(b"test")) self.assertTrue(h.hasHeader(b"Test")) self.assertTrue(h.hasHeader("test")) self.assertTrue(h.hasHeader("Test")) self.assertEqual(h.getRawHeaders("test"), rawValue) self.assertEqual(h.getRawHeaders(b"test"), rawEncodedValue)
def test_initializer(self): """ The header values passed to L{Headers.__init__} can be retrieved via L{Headers.getRawHeaders}. """ h = Headers({'Foo': ['bar']}) self.assertEqual(h.getRawHeaders('foo'), ['bar'])
def test_initializer(self): """ The header values passed to L{Headers.__init__} can be retrieved via L{Headers.getRawHeaders}. """ h = Headers({b"Foo": [b"bar"]}) self.assertEqual(h.getRawHeaders(b"foo"), [b"bar"])
def test_initializer(self): """ The header values passed to L{Headers.__init__} can be retrieved via L{Headers.getRawHeaders}. """ h = Headers({b"Foo": [b"bar"]}) self.assertEqual(h.getRawHeaders(b"foo"), [b"bar"])
def test_initializer(self): """ The header values passed to L{Headers.__init__} can be retrieved via L{Headers.getRawHeaders}. """ h = Headers({b'Foo': [b'bar']}) self.assertEqual(h.getRawHeaders(b'foo'), [b'bar'])
def check_content_type_is(headers: Headers, expected_content_type: str) -> None: """ Check that a set of HTTP headers have a Content-Type header, and that it is the expected value.. Args: headers: headers to check Raises: RequestSendFailed: if the Content-Type header is missing or doesn't match """ content_type_headers = headers.getRawHeaders(b"Content-Type") if content_type_headers is None: raise RequestSendFailed( RuntimeError("No Content-Type header received from remote server"), can_retry=False, ) c_type = content_type_headers[0].decode("ascii") # only the first header val, options = cgi.parse_header(c_type) if val != expected_content_type: raise RequestSendFailed( RuntimeError( f"Remote server sent Content-Type header of '{c_type}', not '{expected_content_type}'", ), can_retry=False, )
def _cache_period_from_headers( headers: Headers, time_now: Callable[[], float] = time.time) -> Optional[float]: cache_controls = _parse_cache_control(headers) if b"no-store" in cache_controls: return 0 if b"max-age" in cache_controls: max_age = cache_controls[b"max-age"] if max_age: try: return int(max_age) except ValueError: pass expires = headers.getRawHeaders(b"expires") if expires is not None: try: expires_date = stringToDatetime(expires[-1]) return expires_date - time_now() except ValueError: # RFC7234 says 'A cache recipient MUST interpret invalid date formats, # especially the value "0", as representing a time in the past (i.e., # "already expired"). return 0 return None
def check_content_type_is_json(headers: Headers) -> None: """ Check that a set of HTTP headers have a Content-Type header, and that it is application/json. Args: headers: headers to check Raises: RequestSendFailed: if the Content-Type header is missing or isn't JSON """ c_type = headers.getRawHeaders(b"Content-Type") if c_type is None: raise RequestSendFailed( RuntimeError("No Content-Type header received from remote server"), can_retry=False, ) c_type = c_type[0].decode("ascii") # only the first header val, options = cgi.parse_header(c_type) if val != "application/json": raise RequestSendFailed( RuntimeError( "Remote server sent Content-Type header of '%s', not 'application/json'" % c_type, ), can_retry=False, )
def test_getRawHeadersDefaultValue(self): """ L{Headers.getRawHeaders} returns the specified default value when no header is found. """ h = Headers() default = object() self.assertIdentical(h.getRawHeaders(b"test", default), default)
def test_getRawHeadersDefaultValue(self): """ L{Headers.getRawHeaders} returns the specified default value when no header is found. """ h = Headers() default = object() self.assertIdentical(h.getRawHeaders("test", default), default)
def test_rawHeadersValueEncoding(self): """ Passing L{unicode} to L{Headers.setRawHeaders} will encode the name as ISO-8859-1 and values as UTF-8. """ h = Headers() h.setRawHeaders(u"\u00E1", [u"\u2603", b"foo"]) self.assertTrue(h.hasHeader(b"\xe1")) self.assertEqual(h.getRawHeaders(b"\xe1"), [b'\xe2\x98\x83', b'foo'])
def get_content_type(headers: Headers) -> Optional[str]: """ Get the content type from the HTTP ``Content-Type`` header. Returns ``None`` if no content-type was set. """ values = headers.getRawHeaders("content-type") or [None] content_type = parse_options_header(values[0])[0] or None return content_type
def test_rawHeadersValueEncoding(self): """ Passing L{unicode} to L{Headers.setRawHeaders} will encode the name as ISO-8859-1 and values as UTF-8. """ h = Headers() h.setRawHeaders("\u00E1", ["\u2603", b"foo"]) self.assertTrue(h.hasHeader(b"\xe1")) self.assertEqual(h.getRawHeaders(b"\xe1"), [b"\xe2\x98\x83", b"foo"])
class _FakeUrllib2Request(object): """ A fake C{urllib2.Request} object for C{cookielib} to work with. @see: U{http://docs.python.org/library/urllib2.html#request-objects} @type uri: C{str} @ivar uri: Request URI. @type headers: L{twisted.web.http_headers.Headers} @ivar headers: Request headers. @type type: C{str} @ivar type: The scheme of the URI. @type host: C{str} @ivar host: The host[:port] of the URI. @since: 11.1 """ def __init__(self, uri): self.uri = uri self.headers = Headers() self.type, rest = splittype(self.uri) self.host, rest = splithost(rest) def has_header(self, header): return self.headers.hasHeader(header) def add_unredirected_header(self, name, value): self.headers.addRawHeader(name, value) def get_full_url(self): return self.uri def get_header(self, name, default=None): headers = self.headers.getRawHeaders(name, default) if headers is not None: return headers[0] return None def get_host(self): return self.host def get_type(self): return self.type def is_unverifiable(self): # In theory this shouldn't be hardcoded. return False
def test_getRawHeaders(self): """ L{Headers.getRawHeaders} returns the values which have been set for a given header. """ h = Headers() h.setRawHeaders("test", ["lemur"]) self.assertEqual(h.getRawHeaders("test"), ["lemur"]) self.assertEqual(h.getRawHeaders("Test"), ["lemur"])
def _parse_cache_control(headers: Headers) -> Dict[bytes, Optional[bytes]]: cache_controls = {} for hdr in headers.getRawHeaders(b"cache-control", []): for directive in hdr.split(b","): splits = [x.strip() for x in directive.split(b"=", 1)] k = splits[0].lower() v = splits[1] if len(splits) > 1 else None cache_controls[k] = v return cache_controls
def test_initializer(self): """ The header values passed to L{Headers.__init__} can be retrieved via L{Headers.getRawHeaders}. If a L{bytes} argument is given, it returns L{bytes} values, and if a L{unicode} argument is given, it returns L{unicode} values. Both are the same header value, just encoded or decoded. """ h = Headers({u'Foo': [u'bar']}) self.assertEqual(h.getRawHeaders(b'foo'), [b'bar']) self.assertEqual(h.getRawHeaders(u'foo'), [u'bar'])
def request(self, method, uri, headers, bodyProducer): if headers is None: headers = Headers() else: headers = headers.copy() contentType = headers.getRawHeaders('content-type', [""])[0] date = headers.getRawHeaders('date', [""])[0] or self._generateRequestDate(uri) headers.setRawHeaders('date', [date]) uri_origin_form = URI.fromBytes(uri).originForm contentMD5 = headers.getRawHeaders('content-md5', [""])[0] if not contentMD5 and bodyProducer is not None: r = getattr(self.agent, '_reactor') or reactor bodyConsumer = StringConsumer(callLater=r.callLater) yield bodyProducer.startProducing(bodyConsumer) body = bodyConsumer.value() bodyProducer = StringBodyProducer(body) if body: contentMD5 = binascii.b2a_base64( hashlib.md5(body).digest()).strip() headers.addRawHeader('content-md5', contentMD5) sts = "\n".join([ method, contentType or "", contentMD5, date or "", uri_origin_form ]) mac = hmac.new(self.secretKey, sts, digestmod=hashlib.sha1).digest() encodedMAC = binascii.b2a_base64(mac).strip() auth_header = "AuthHMAC {0}:{1}".format(self.accessKey, encodedMAC) headers.addRawHeader('authorization', auth_header) d = yield self.agent.request(method, uri, headers, bodyProducer) self._handleResponseDate(uri, d) defer.returnValue(d)
def test_copy(self): """ L{Headers.copy} creates a new independant copy of an existing L{Headers} instance, allowing future modifications without impacts between the copies. """ h = Headers() h.setRawHeaders('test', ['foo']) i = h.copy() self.assertEquals(i.getRawHeaders('test'), ['foo']) h.addRawHeader('test', 'bar') self.assertEquals(i.getRawHeaders('test'), ['foo']) i.addRawHeader('test', 'baz') self.assertEquals(h.getRawHeaders('test'), ['foo', 'bar'])
class _FakeUrllib2Request(object): """ A fake C{urllib2.Request} object for C{cookielib} to work with. @see: U{http://docs.python.org/library/urllib2.html#request-objects} @type uri: C{str} @ivar uri: Request URI. @type headers: L{twisted.web.http_headers.Headers} @ivar headers: Request headers. @type type: C{str} @ivar type: The scheme of the URI. @type host: C{str} @ivar host: The host[:port] of the URI. @since: 11.1 """ def __init__(self, uri): self.uri = uri self.headers = Headers() self.type, rest = splittype(self.uri) self.host, rest = splithost(rest) def has_header(self, header): return self.headers.hasHeader(header) def add_unredirected_header(self, name, value): self.headers.addRawHeader(name, value) def get_full_url(self): return self.uri def get_header(self, name, default=None): headers = self.headers.getRawHeaders(name, default) if headers is not None: return headers[0] return None def get_host(self): return self.host def get_type(self): return self.type def is_unverifiable(self): # In theory this shouldn't be hardcoded. return False
def test_copy(self): """ L{Headers.copy} creates a new independant copy of an existing L{Headers} instance, allowing future modifications without impacts between the copies. """ h = Headers() h.setRawHeaders(b"test", [b"foo"]) i = h.copy() self.assertEqual(i.getRawHeaders(b"test"), [b"foo"]) h.addRawHeader(b"test", b"bar") self.assertEqual(i.getRawHeaders(b"test"), [b"foo"]) i.addRawHeader(b"test", b"baz") self.assertEqual(h.getRawHeaders(b"test"), [b"foo", b"bar"])
def test_copy(self): """ L{Headers.copy} creates a new independant copy of an existing L{Headers} instance, allowing future modifications without impacts between the copies. """ h = Headers() h.setRawHeaders(b'test', [b'foo']) i = h.copy() self.assertEqual(i.getRawHeaders(b'test'), [b'foo']) h.addRawHeader(b'test', b'bar') self.assertEqual(i.getRawHeaders(b'test'), [b'foo']) i.addRawHeader(b'test', b'baz') self.assertEqual(h.getRawHeaders(b'test'), [b'foo', b'bar'])
def test_copy(self): """ L{Headers.copy} creates a new independent copy of an existing L{Headers} instance, allowing future modifications without impacts between the copies. """ h = Headers() h.setRawHeaders(b"test", [b"foo"]) i = h.copy() self.assertEqual(i.getRawHeaders(b"test"), [b"foo"]) h.addRawHeader(b"test", b"bar") self.assertEqual(i.getRawHeaders(b"test"), [b"foo"]) i.addRawHeader(b"test", b"baz") self.assertEqual(h.getRawHeaders(b"test"), [b"foo", b"bar"])
def request(self, method, uri, headers, bodyProducer): if headers is None: headers = Headers() else: headers = headers.copy() contentType = headers.getRawHeaders('content-type', [""])[0] date = headers.getRawHeaders('date', [""])[0] or self._generateRequestDate(uri) headers.setRawHeaders('date', [date]) uri_origin_form = URI.fromBytes(uri).originForm contentMD5 = headers.getRawHeaders('content-md5', [""])[0] if not contentMD5 and bodyProducer is not None: r = getattr(self.agent, '_reactor') or reactor bodyConsumer = StringConsumer(callLater=r.callLater) yield bodyProducer.startProducing(bodyConsumer) body = bodyConsumer.value() bodyProducer = StringBodyProducer(body) if body: contentMD5 = binascii.b2a_base64(hashlib.md5(body).digest()).strip() headers.addRawHeader('content-md5', contentMD5) sts = "\n".join([method, contentType or "", contentMD5, date or "", uri_origin_form]) mac = hmac.new(self.secretKey, sts, digestmod=hashlib.sha1).digest() encodedMAC = binascii.b2a_base64(mac).strip() auth_header = "AuthHMAC {0}:{1}".format(self.accessKey, encodedMAC) headers.addRawHeader('authorization', auth_header) d = yield self.agent.request(method, uri, headers, bodyProducer) self._handleResponseDate(uri, d) defer.returnValue(d)
class FakeRequest(object): """ A fake request suitable for use in place of C{twisted.web.http.Request}. @param method: The HTTP method being invoked such as C{GET} or C{POST}. """ def __init__(self, method): self.method = method self.requestHeaders = Headers() self.responseHeaders = Headers() self._fluidDB_reqid = 'xxx' self.finished = False def finish(self): """ Indicate that all response data has been written to this request. """ self.finished = True def getResponseHeader(self, name): """ Get the value of an HTTP response header. @param name: The name of the HTTP header to retrieve a value for. @return: The value set for the header or C{None} if a value is not available. """ value = self.responseHeaders.getRawHeaders(name) if value is not None: return value[-1] def setHeader(self, name, value): """ Set an HTTP header to include in the response returned to the client. @param name: The name of the header to set. @param value: The value to set. """ self.responseHeaders.setRawHeaders(name, [value]) def setResponseCode(self, code): """ Set the HTTP response code to return to the client. @param code: An HTTP response code as defined in C{twisted.web.http}. """ self.status = code
def test_nameEncoding(self): """ Passing L{unicode} to any function that takes a header name will encode said header name as ISO-8859-1. """ h = Headers() # We set it using a Unicode string. h.setRawHeaders(u"\u00E1", [b"foo"]) # It's encoded to the ISO-8859-1 value, which we can use to access it self.assertTrue(h.hasHeader(b"\xe1")) self.assertEqual(h.getRawHeaders(b"\xe1"), [b'foo']) # We can still access it using the Unicode string.. self.assertTrue(h.hasHeader(u"\u00E1"))
def test_nameEncoding(self): """ Passing L{unicode} to any function that takes a header name will encode said header name as ISO-8859-1. """ h = Headers() # We set it using a Unicode string. h.setRawHeaders("\u00E1", [b"foo"]) # It's encoded to the ISO-8859-1 value, which we can use to access it self.assertTrue(h.hasHeader(b"\xe1")) self.assertEqual(h.getRawHeaders(b"\xe1"), [b"foo"]) # We can still access it using the Unicode string.. self.assertTrue(h.hasHeader("\u00E1"))
class FakeRequest(object): """ A fake request suitable for use in place of C{twisted.web.http.Request}. @param method: The HTTP method being invoked such as C{GET} or C{POST}. """ def __init__(self, method): self.method = method self.requestHeaders = Headers() self.responseHeaders = Headers() self._fluidDB_reqid = 'xxx' self.finished = False def finish(self): """ Indicate that all response data has been written to this request. """ self.finished = True def getResponseHeader(self, name): """ Get the value of an HTTP response header. @param name: The name of the HTTP header to retrieve a value for. @return: The value set for the header or C{None} if a value is not available. """ value = self.responseHeaders.getRawHeaders(name) if value is not None: return value[-1] def setHeader(self, name, value): """ Set an HTTP header to include in the response returned to the client. @param name: The name of the header to set. @param value: The value to set. """ self.responseHeaders.setRawHeaders(name, [value]) def setResponseCode(self, code): """ Set the HTTP response code to return to the client. @param code: An HTTP response code as defined in C{twisted.web.http}. """ self.status = code
class _FakeUrllib2Request(object): """ A fake C{urllib2.Request} object for C{cookielib} to work with. @type uri: C{str} @ivar uri: Request URI. @type headers: L{twisted.web.http_headers.Headers} @ivar headers: Request headers. @since: 11.1 """ def __init__(self, uri): self.uri = uri self.headers = Headers() def has_header(self, header): return self.headers.hasHeader(header) def add_unredirected_header(self, name, value): self.headers.addRawHeader(name, value) def get_full_url(self): return self.uri def get_header(self, name, default=None): headers = self.headers.getRawHeaders(name, default) if headers is not None: return headers[0] return None def is_unverifiable(self): # In theory this shouldn't be hardcoded. return False
class FakeRequest(Componentized): """ Implementation of L{inevow.IRequest} which is convenient to use in unit tests. @ivar lastModified: The value passed to L{setLastModified} or C{None} if that method has not been called. @type accumulator: C{str} @ivar accumulator: The bytes written to the response body. @type deferred: L{Deferred} @ivar deferred: The deferred which represents rendering of the response to this request. This is basically an implementation detail of L{NevowRequest}. Application code should probably never use this. @ivar _appRootURL: C{None} or the object passed to L{rememberRootURL}. """ implements(inevow.IRequest) fields = None failure = None context = None redirected_to = None lastModified = None content = "" method = 'GET' code = http.OK deferred = None accumulator = '' _appRootURL = None def __init__(self, headers=None, args=None, avatar=None, uri='/', currentSegments=None, cookies=None, user="", password="", isSecure=False): """ Create a FakeRequest instance. @param headers: dict of request headers @param args: dict of args @param avatar: avatar to pass to the FakeSession instance @param uri: request URI @param currentSegments: list of segments that have "already been located" @param cookies: dict of cookies @param user: username (like in http auth) @param password: password (like in http auth) @param isSecure: whether this request represents an HTTPS url """ Componentized.__init__(self) self.uri = uri if not uri.startswith('/'): raise ValueError('uri must be relative with absolute path') self.path = uri self.prepath = [] postpath = uri.split('?')[0] assert postpath.startswith('/') self.postpath = postpath[1:].split('/') if currentSegments is not None: for seg in currentSegments: assert seg == self.postpath[0] self.prepath.append(self.postpath.pop(0)) else: self.prepath.append('') self.responseHeaders = Headers() self.args = args or {} self.sess = FakeSession(avatar) self.site = FakeSite() self.requestHeaders = Headers() if headers: for k, v in headers.iteritems(): self.requestHeaders.setRawHeaders(k, [v]) if cookies is not None: self.cookies = cookies else: self.cookies = {} self.user = user self.password = password self.secure = isSecure self.deferred = defer.Deferred() def URLPath(self): from nevow import url return url.URL.fromContext(self) def getSession(self): return self.sess def registerProducer(self, producer, streaming): """ Synchronously cause the given producer to produce all of its data. This will not work with push producers. Do not use it with them. """ keepGoing = [None] self.unregisterProducer = keepGoing.pop while keepGoing: producer.resumeProducing() del self.unregisterProducer def v(): def get(self): return self.accumulator return get, v = property(*v()) def write(self, bytes): """ Accumulate the given bytes as part of the response body. @type bytes: C{str} """ self.accumulator += bytes finished = False def finishRequest(self, success): self.finished = True def finish(self): self.deferred.callback('') def getHeader(self, key): return self.requestHeaders.getRawHeaders(key, [None])[0] def setHeader(self, key, val): self.responseHeaders.setRawHeaders(key, [val]) def redirect(self, url): self.redirected_to = url def processingFailed(self, f): self.failure = f def setResponseCode(self, code): self.code = code def setLastModified(self, when): self.lastModified = when def prePathURL(self): """ The absolute URL up until the last handled segment of this request. @rtype: C{str}. """ return 'http://%s/%s' % (self.getHeader('host') or 'localhost', '/'.join(self.prepath)) def getClientIP(self): return '127.0.0.1' def addCookie(self, k, v, expires=None, domain=None, path=None, max_age=None, comment=None, secure=None): """ Set a cookie for use in subsequent requests. """ self.cookies[k] = v def getCookie(self, k): """ Fetch a cookie previously set. """ return self.cookies.get(k) def getUser(self): """ Returns the HTTP auth username. """ return self.user def getPassword(self): """ Returns the HTTP auth password. """ return self.password def getRootURL(self): """ Return the previously remembered URL. """ return self._appRootURL def rememberRootURL(self, url=None): """ For compatibility with appserver.NevowRequest. """ if url is None: raise NotImplementedError( "Default URL remembering logic is not implemented.") self._appRootURL = url def isSecure(self): """ Returns whether this is an HTTPS request or not. """ return self.secure def _warnHeaders(self, old, new): """ Emit a warning related to use of one of the deprecated C{headers} or C{received_headers} attributes. @param old: The name of the deprecated attribute to which the warning pertains. @param new: The name of the preferred attribute which replaces the old attribute. """ warnings.warn( category=DeprecationWarning, message=( "nevow.testutil.FakeRequest.%(old)s was deprecated in " "Nevow 0.13.0: Please use nevow.testutil.FakeRequest." "%(new)s instead." % dict(old=old, new=new)), stacklevel=3) @property def headers(self): """ Transform the L{Headers}-style C{responseHeaders} attribute into a deprecated C{dict}-style C{headers} attribute. """ self._warnHeaders("headers", "responseHeaders") return appserver._DictHeaders(self.responseHeaders) @property def received_headers(self): """ Transform the L{Headers}-style C{requestHeaders} attribute into a deprecated C{dict}-style C{received_headers} attribute. """ self._warnHeaders("received_headers", "requestHeaders") return appserver._DictHeaders(self.requestHeaders)
class FakeRequest(object): """ I pretend to be an HTTP request, with a handful of required methods used by Twisted resource classes (e.g., write, finish, setResponseCode). """ _fluidDB_reqid = None body = '' def __init__(self, method, d=None, headers=None, hostname=None): """ Initialize. d (if not None) is a deferred that will fire with the body of the response when request.finish is called. @param d: A C{Deferred} instance to fire when the request is finished. @param headers: a C{dict} of headers and values for the request. """ self.method = method self.d = d self.uri = None self.requestHeaders = Headers(headers) self.responseHeaders = Headers() self.args = {'verb': [method]} self.hostname = hostname self.content = StringIO.StringIO() def isSecure(self): """ Mark the request as not being secure (i.e., not HTTPS). The isSecure is used by the FluidDB wsfe to construct the Location header in the response to a POST. @return: A constant, C{False}, seeing as we don't care about the return value, only that we provide the method. """ return False def finish(self): """ Indicate that the request has been fully serviced. Send the response body (as accumulated by self.write) back to the deferred (if any) we were passed in __init__. @return: C{None}. """ if self.d: self.d.callback(self.body) def write(self, data): """ Fake the writing of data back to the client. Instead, we accumuluate it so as to deliver it to our test client when self.finish is called. @param data: A C{str} of response payload data. @return: C{None}. """ self.body += data def getHeader(self, key): """ Get a header from the request. This is copied from Twisted's twisted.web.http.Request class. @param key: A C{str} indicating the header to return. @return: The C{str} header value from the request if it exists, else C{None}. """ value = self.requestHeaders.getRawHeaders(key) if value is not None: return value[-1] def getResponseHeader(self, key): """ Get a header from the response. @param key: A C{str} indicating the header to return. @return: The C{str} header value from the response if it exists, else C{None}. """ value = self.responseHeaders.getRawHeaders(key) if value is not None: return value[-1] def setHeader(self, name, value): """ Set a header for the HTTP response. This is copied from Twisted's twisted.web.http.Request class. @param name: A C{str} indicating the header to set. @param value: A C{str} values for the header. @return: C{None}. """ self.responseHeaders.setRawHeaders(name, [value]) def getAllHeaders(self): """ Get all the request headers. This is copied from Twisted's twisted.web.http.Request class. @return: A C{dict} of request header name -> value. """ headers = {} for k, v in self.requestHeaders.getAllRawHeaders(): headers[k.lower()] = v[-1] return headers def setResponseCode(self, code): """ Set the response status code. @param code: An HTTP status code. @return: C{None}. """ self.status = code def notifyFinish(self): """ Return a C{twisted.internet.Deferred} that fires when the request finishes or errors if the client disconnects. Note that this method is needed as resource.py calls it, but we do not need to actually fire the returned deferred for our tests (which cause synchronous exceptions to immediately be returned to the request errback). @return: C{twisted.internet.Deferred} """ return defer.succeed(None) def getRequestHostname(self): """ Return the hostname that was used for issuing this request. """ return self.hostname
class T2WRequest(http.Request): """ Used by Tor2webProxy to implement a simple web proxy. """ def __init__(self, channel, queued, reactor=reactor): """ Method overridden to change some part of proxy.Request and of the base http.Request """ self.reactor = reactor self.notifications = [] self.channel = channel self.queued = queued self.requestHeaders = Headers() self.received_cookies = {} self.responseHeaders = Headers() self.cookies = [] # outgoing cookies self.bodyProducer = BodyProducer() self.proxy_d = None self.proxy_response = None self.stream = '' self.header_injected = False if queued: self.transport = StringTransport() else: self.transport = self.channel.transport self.obj = Tor2webObj() self.var = Storage() self.var['version'] = VERSION self.var['basehost'] = config.basehost self.var['errorcode'] = None self.html = False self.decoderGzip = None self.encoderGzip = None self.pool = pool def _cleanup(self): """ Method overridden to avoid self.content actions. """ if self.producer: log.err( RuntimeError("Producer was not unregistered for %s" % self.uri)) self.unregisterProducer() self.channel.requestDone(self) del self.channel for d in self.notifications: d.callback(None) self.notifications = [] def getRequestHostname(self): """ Function overload to fix ipv6 bug: http://twistedmatrix.com/trac/ticket/6014 """ host = self.getHeader(b'host') if host: if host[0] == '[': return host.split(']', 1)[0] + "]" return networkString(host.split(':', 1)[0]) return networkString(self.getHost().host) def forwardData(self, data, end=False): if not self.startedWriting: if self.obj.client_supports_gzip: self.setHeader(b'content-encoding', b'gzip') if data != '' and end: self.setHeader(b'content-length', intToBytes(len(data))) if data != '': try: self.write(data) except: pass def requestReceived(self, command, path, version): """ Method overridden to reduce the function actions """ self.method, self.uri = command, path self.clientproto = version # cache the client and server information, we'll need this later to be # serialized and sent with the request so CGIs will work remotely self.client = self.channel.transport.getPeer() self.host = self.channel.transport.getHost() self.process() def add_banner(self, banner, data): """ Inject tor2web banner inside the returned page """ return str(data.group(1)) + str(banner) @defer.inlineCallbacks def handleFixPart(self, data): if self.obj.server_response_is_gzip: data = self.unzip(data) data = self.stream + data if len(data) >= 1000: if not self.header_injected and data.find("<body") != -1: banner = yield flattenString(self, templates['banner.tpl']) data = re.sub(rexp['body'], partial(self.add_banner, banner), data) self.header_injected = True data = re_sub(rexp['t2w'], r'https://\2.' + config.basehost, data) self.forwardData(self.handleCleartextForwardPart(data[:-500])) self.stream = data[-500:] else: self.stream = data @defer.inlineCallbacks def handleFixEnd(self, data): if self.obj.server_response_is_gzip: data = self.unzip(data, True) data = self.stream + data if not self.header_injected and data.find("<body") != -1: banner = yield flattenString(self, templates['banner.tpl']) data = re.sub(rexp['body'], partial(self.add_banner, banner), data) self.header_injected = True data = re_sub(rexp['t2w'], r'https://\2.' + config.basehost, data) data = self.handleCleartextForwardPart(data, True) self.forwardData(data, True) self.stream = '' try: self.finish() except: pass def handleGzippedForwardPart(self, data, end=False): if not self.obj.client_supports_gzip: data = self.unzip(data, end) return data def handleCleartextForwardPart(self, data, end=False): if self.obj.client_supports_gzip: data = self.zip(data, end) return data def handleForwardPart(self, data): if self.obj.server_response_is_gzip: data = self.handleGzippedForwardPart(data) else: data = self.handleCleartextForwardPart(data) self.forwardData(data) def handleForwardEnd(self, data): if self.obj.server_response_is_gzip: data = self.handleGzippedForwardPart(data, True) else: data = self.handleCleartextForwardPart(data, True) self.forwardData(data, True) try: self.finish() except: pass def contentFinish(self, data): if self.obj.client_supports_gzip: self.setHeader(b'content-encoding', b'gzip') data = self.zip(data, True) self.setHeader(b'content-length', intToBytes(len(data))) try: self.write(data) self.finish() except: pass def sendError(self, error=500, errortemplate='error_generic.tpl'): self.setResponseCode(error) self.var['errorcode'] = error return flattenString(self, templates[errortemplate]).addCallback( self.contentFinish) def handleError(self, failure): if type(failure.value) is SOCKSError: self.setResponseCode(404) self.var['errorcode'] = failure.value.code if failure.value.code in SOCKS_errors: return flattenString( self, templates[SOCKS_errors[failure.value.code]]).addCallback( self.contentFinish) else: return flattenString( self, templates[SOCKS_errors[0x00]]).addCallback( self.contentFinish) else: self.sendError() def unzip(self, data, end=False): data1 = data2 = '' try: if self.decoderGzip == None: self.decoderGzip = zlib.decompressobj(16 + zlib.MAX_WBITS) if data != '': data1 = self.decoderGzip.decompress(data) if end: data2 = self.decoderGzip.flush() except: pass return data1 + data2 def zip(self, data, end=False): data1 = data2 = '' try: if self.encoderGzip == None: self.encoderGzip = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS) if data != '': data1 = self.encoderGzip.compress(data) if end: data2 = self.encoderGzip.flush() except: pass return data1 + data2 @defer.inlineCallbacks def process(self): content = "" request = Storage() request.headers = self.requestHeaders request.host = self.getRequestHostname() request.uri = self.uri content_length = self.getHeader(b'content-length') transfer_encoding = self.getHeader(b'transfer-encoding') staticpath = request.uri staticpath = re.sub('\/$', '/index.html', staticpath) staticpath = re.sub('^(/antanistaticmap/)?', '', staticpath) staticpath = re.sub('^/', '', staticpath) resource_is_local = isIPAddress(request.host) or \ isIPv6Address(request.host) or \ request.uri == '/robots.txt' or \ request.uri.startswith('/antanistaticmap/') if content_length is not None: self.bodyProducer.length = int(content_length) producer = self.bodyProducer request.headers.removeHeader(b'content-length') elif transfer_encoding is not None: producer = self.bodyProducer request.headers.removeHeader(b'transfer-encoding') else: producer = None if config.mirror is not None: if config.basehost in config.mirror: config.mirror.remove(config.basehost) self.var['mirror'] = choice(config.mirror) # we serve contents only over https if not self.isSecure(): self.redirect("https://" + request.host + request.uri) self.finish() return # 0: Request admission control stage # we try to deny some ua/crawlers regardless the request is (valid or not) / (local or not) # we deny EVERY request to known user agents reconized with pattern matching if config.blockcrawl and request.headers.getRawHeaders( b'user-agent') != None: for ua in t2w.blocked_ua: check = request.headers.getRawHeaders(b'user-agent')[0].lower() if re.match(ua, check): self.sendError(403, "error_blocked_ua.tpl") defer.returnValue(NOT_DONE_YET) # 1: Client capability assessment stage if request.headers.getRawHeaders(b'accept-encoding') != None: if re.search('gzip', request.headers.getRawHeaders(b'accept-encoding')[0]): self.obj.client_supports_gzip = True # 2: Content delivery stage # we need to verify if the requested resource is local (/antanistaticmap/*) or remote # because some checks must be done only for remote requests; # in fact local content is always served (css, js, and png in fact are used in errors) if resource_is_local: # the requested resource is local, we deliver it directly try: if staticpath == "notification": ################################################################# # Here we need to parse POST data in x-www-form-urlencoded format ################################################################# content_receiver = BodyReceiver(defer.Deferred()) self.bodyProducer.startProducing(content_receiver) yield self.bodyProducer.finished content = ''.join(content_receiver._data) args = {} ctype = self.requestHeaders.getRawHeaders(b'content-type') if ctype is not None: ctype = ctype[0] if self.method == b"POST" and ctype: mfd = b'multipart/form-data' key, pdict = parse_header(ctype) if key == b'application/x-www-form-urlencoded': args.update(parse_qs(content, 1)) ################################################################# if 'by' in args and 'url' in args and 'comment' in args: tmp = [] tmp.append("From: Tor2web Node %s.%s <%s>\n" % (config.nodename, config.basehost, config.smtpmail)) tmp.append("To: %s\n" % (config.smtpmailto_notifications)) tmp.append( "Subject: Tor2web Node (IPv4 %s, IPv6 %s): notification for %s\n" % (config.listen_ipv4, config.listen_ipv6, args['url'][0])) tmp.append( "Content-Type: text/plain; charset=ISO-8859-1\n") tmp.append("Content-Transfer-Encoding: 8bit\n\n") tmp.append("BY: %s\n" % (args['by'][0])) tmp.append("URL: %s\n" % (args['url'][0])) tmp.append("COMMENT: %s\n" % (args['comment'][0])) message = StringIO(''.join(tmp)) try: sendmail(config.smtpuser, config.smtppass, config.smtpmail, config.smtpmailto_notifications, message, config.smtpdomain, config.smtpport) except: pass else: if type(antanistaticmap[staticpath]) == str: filename, ext = os.path.splitext(staticpath) self.setHeader(b'content-type', mimetypes.types_map[ext]) content = antanistaticmap[staticpath] defer.returnValue(self.contentFinish(content)) elif type(antanistaticmap[staticpath]) == PageTemplate: defer.returnValue( flattenString( self, antanistaticmap[staticpath]).addCallback( self.contentFinish)) except: pass self.sendError(404) defer.returnValue(NOT_DONE_YET) else: self.obj.uri = request.uri if not request.host: self.sendError(406, 'error_invalid_hostname.tpl') defer.returnValue(NOT_DONE_YET) if config.mode == "TRANSLATION": self.obj.onion = config.onion else: self.obj.onion = request.host.split(".")[0] + ".onion" log.msg("detected <onion_url>.tor2web Hostname: %s" % self.obj.onion) if not verify_onion(self.obj.onion): self.sendError(406, 'error_invalid_hostname.tpl') defer.returnValue(NOT_DONE_YET) if config.mode == "ACCESSLIST": if self.obj.onion not in t2w.accesslist: self.sendError(403, 'error_hs_completely_blocked.tpl') defer.returnValue(NOT_DONE_YET) elif config.mode == "BLACKLIST": if hashlib.md5( self.obj.onion).hexdigest() in t2w.accesslist: self.sendError(403, 'error_hs_completely_blocked.tpl') defer.returnValue(NOT_DONE_YET) if hashlib.md5(self.obj.onion + self.obj.uri).hexdigest() in t2w.accesslist: self.sendError(403, 'error_hs_specific_page_blocked.tpl') defer.returnValue(NOT_DONE_YET) # we need to verify if the user is using tor; # on this condition it's better to redirect on the .onion if self.getClientIP() in t2w.TorExitNodes: self.redirect("http://" + self.obj.onion + request.uri) try: self.finish() except: pass return # Avoid image hotlinking if request.uri.lower().endswith(('gif', 'jpg', 'png')): if request.headers.getRawHeaders( b'referer' ) != None and not config.basehost in request.headers.getRawHeaders( b'referer')[0].lower(): self.sendError(403) defer.returnValue(NOT_DONE_YET) # the requested resource is remote, we act as proxy t2w.process_request(self.obj, request) parsed = urlparse(self.obj.address) self.var['address'] = self.obj.address self.var['onion'] = self.obj.onion.replace(".onion", "") self.var['path'] = parsed[2] + '?' + parsed[3] agent = Agent(reactor, sockhost=config.sockshost, sockport=config.socksport, pool=self.pool) self.proxy_d = agent.request(self.method, 's' + self.obj.address, self.obj.headers, bodyProducer=producer) self.proxy_d.addCallback(self.cbResponse) self.proxy_d.addErrback(self.handleError) defer.returnValue(NOT_DONE_YET) def cbResponse(self, response): self.proxy_response = response if int(response.code) >= 600 and int(response.code) <= 699: self.setResponseCode(500) self.var['errorcode'] = int(response.code) - 600 if self.var['errorcode'] in SOCKS_errors: return flattenString( self, templates[SOCKS_errors[ self.var['errorcode']]]).addCallback( self.contentFinish) else: return flattenString( self, templates[SOCKS_errors[0x00]]).addCallback( self.contentFinish) self.setResponseCode(response.code) self.processResponseHeaders(response.headers) if (response.length is not 0): finished = defer.Deferred() if self.obj.html: response.deliverBody(BodyStreamer(self.handleFixPart, finished)) finished.addCallback(self.handleFixEnd) else: response.deliverBody( BodyStreamer(self.handleForwardPart, finished)) finished.addCallback(self.handleForwardEnd) return finished else: self.contentFinish('') return defer.succeed def handleHeader(self, key, values): keyLower = key.lower() # some headers does not allow multiple occurrences # in case of multiple occurrences we evaluate only the first valueLower = values[0].lower() if keyLower == 'transfer-encoding' and valueLower == 'chunked': return elif keyLower == 'content-encoding' and valueLower == 'gzip': self.obj.server_response_is_gzip = True return elif keyLower == 'content-type' and re.search('text/html', valueLower): self.obj.html = True elif keyLower == 'content-length': self.receivedContentLen = valueLower return elif keyLower == 'cache-control': return if keyLower in ('location'): fixed_values = [] for value in values: value = re_sub(rexp['t2w'], r'https://\2.' + config.basehost, value) fixed_values.append(value) values = fixed_values self.responseHeaders.setRawHeaders(key, values) def handleEndHeaders(self): self.setHeader(b'cache-control', b'no-cache') self.setHeader(b'strict-transport-security', b'max-age=31536000') def processResponseHeaders(self, headers): for key, values in headers.getAllRawHeaders(): self.handleHeader(key, values) self.handleEndHeaders() def connectionLost(self, reason): try: if self.proxy_d: self.proxy_d.cancel() except: pass try: if self.proxy_response: self.proxy_response._transport.stopProducing() except: pass http.Request.connectionLost(self, reason)
class DummyRequest(object): """ Represents a dummy or fake request. See L{twisted.web.server.Request}. @ivar _finishedDeferreds: C{None} or a C{list} of L{Deferreds} which will be called back with C{None} when C{finish} is called or which will be errbacked if C{processingFailed} is called. @type requestheaders: C{Headers} @ivar requestheaders: A Headers instance that stores values for all request headers. @type responseHeaders: C{Headers} @ivar responseHeaders: A Headers instance that stores values for all response headers. @type responseCode: C{int} @ivar responseCode: The response code which was passed to C{setResponseCode}. @type written: C{list} of C{bytes} @ivar written: The bytes which have been written to the request. """ uri = b'http://dummy/' method = b'GET' client = None def registerProducer(self, prod,s): self.go = 1 while self.go: prod.resumeProducing() def unregisterProducer(self): self.go = 0 def __init__(self, postpath, session=None): self.sitepath = [] self.written = [] self.finished = 0 self.postpath = postpath self.prepath = [] self.session = None self.protoSession = session or Session(0, self) self.args = {} self.requestHeaders = Headers() self.responseHeaders = Headers() self.responseCode = None self._finishedDeferreds = [] self._serverName = b"dummy" self.clientproto = b"HTTP/1.0" def getAllHeaders(self): """ Return dictionary mapping the names of all received headers to the last value received for each. Since this method does not return all header information, C{self.requestHeaders.getAllRawHeaders()} may be preferred. NOTE: This function is a direct copy of C{twisted.web.http.Request.getAllRawHeaders}. """ headers = {} for k, v in self.requestHeaders.getAllRawHeaders(): headers[k.lower()] = v[-1] return headers def getHeader(self, name): """ Retrieve the value of a request header. @type name: C{bytes} @param name: The name of the request header for which to retrieve the value. Header names are compared case-insensitively. @rtype: C{bytes} or L{NoneType} @return: The value of the specified request header. """ return self.requestHeaders.getRawHeaders(name.lower(), [None])[0] def setHeader(self, name, value): """TODO: make this assert on write() if the header is content-length """ self.responseHeaders.addRawHeader(name, value) def getSession(self): if self.session: return self.session assert not self.written, "Session cannot be requested after data has been written." self.session = self.protoSession return self.session def render(self, resource): """ Render the given resource as a response to this request. This implementation only handles a few of the most common behaviors of resources. It can handle a render method that returns a string or C{NOT_DONE_YET}. It doesn't know anything about the semantics of request methods (eg HEAD) nor how to set any particular headers. Basically, it's largely broken, but sufficient for some tests at least. It should B{not} be expanded to do all the same stuff L{Request} does. Instead, L{DummyRequest} should be phased out and L{Request} (or some other real code factored in a different way) used. """ result = resource.render(self) if result is NOT_DONE_YET: return self.write(result) self.finish() def write(self, data): if not isinstance(data, bytes): raise TypeError("write() only accepts bytes") self.written.append(data) def notifyFinish(self): """ Return a L{Deferred} which is called back with C{None} when the request is finished. This will probably only work if you haven't called C{finish} yet. """ finished = Deferred() self._finishedDeferreds.append(finished) return finished def finish(self): """ Record that the request is finished and callback and L{Deferred}s waiting for notification of this. """ self.finished = self.finished + 1 if self._finishedDeferreds is not None: observers = self._finishedDeferreds self._finishedDeferreds = None for obs in observers: obs.callback(None) def processingFailed(self, reason): """ Errback and L{Deferreds} waiting for finish notification. """ if self._finishedDeferreds is not None: observers = self._finishedDeferreds self._finishedDeferreds = None for obs in observers: obs.errback(reason) def addArg(self, name, value): self.args[name] = [value] def setResponseCode(self, code, message=None): """ Set the HTTP status response code, but takes care that this is called before any data is written. """ assert not self.written, "Response code cannot be set after data has been written: %s." % "@@@@".join(self.written) self.responseCode = code self.responseMessage = message def setLastModified(self, when): assert not self.written, "Last-Modified cannot be set after data has been written: %s." % "@@@@".join(self.written) def setETag(self, tag): assert not self.written, "ETag cannot be set after data has been written: %s." % "@@@@".join(self.written) def getClientIP(self): """ Return the IPv4 address of the client which made this request, if there is one, otherwise C{None}. """ if isinstance(self.client, IPv4Address): return self.client.host return None def getRequestHostname(self): """ Get a dummy hostname associated to the HTTP request. @rtype: C{bytes} @returns: a dummy hostname """ return self._serverName def getHost(self): """ Get a dummy transport's host. @rtype: C{IPv4Address} @returns: a dummy transport's host """ return IPv4Address('TCP', '127.0.0.1', 80) def setHost(self, host, port, ssl=0): """ Change the host and port the request thinks it's using. @type host: C{bytes} @param host: The value to which to change the host header. @type ssl: C{bool} @param ssl: A flag which, if C{True}, indicates that the request is considered secure (if C{True}, L{isSecure} will return C{True}). """ self._forceSSL = ssl # set first so isSecure will work if self.isSecure(): default = 443 else: default = 80 if port == default: hostHeader = host else: hostHeader = host + b":" + intToBytes(port) self.requestHeaders.addRawHeader(b"host", hostHeader) def getClient(self): """ Get the client's IP address, if it has one. @return: The same value as C{getClientIP}. @rtype: L{bytes} """ return self.getClientIP() def redirect(self, url): """ Utility function that does a redirect. The request should have finish() called after this. """ self.setResponseCode(FOUND) self.setHeader(b"location", url)
class FakeRequest(object): """ A fake C{twisted.web.http.Request} implementation, suitable for use in tests. """ def __init__(self, args=None, postpath=None, prepath=None, path=None, uri=None, method='GET', headers=None, body=''): self.args = {} if args is None else args self.written = StringIO() self.finished = False self.code = None self.method = method self.path = path self.postpath = postpath self.prepath = prepath self.requestHeaders = Headers({}) if headers is None else headers self.responseHeaders = Headers({}) self.uri = uri self._fluidDB_reqid = requestId() self.content = StringIO(body) def write(self, content): """Write data as a result of an HTTP request. @param content: A C{str} containing the bytes to send as part of the response body. """ if not isinstance(content, str): raise RuntimeError('Only strings can be written.') self.written.write(content) def finish(self): """Indicate that all response data has been written to this request.""" if self.code is None: self.code = 200 self.finished = True def setResponseCode(self, code): """Set the HTTP response code. @param code: An C{int} HTTP status code. """ self.code = code def setHeader(self, key, value): """Set an HTTP response header. @param key: The name of the response header. @param value: The value for the response header. """ self.responseHeaders.setRawHeaders(key, [value]) def getHeader(self, key): """ Get a header from the request. This is copied from Twisted's C{twisted.web.http.Request} class. @param key: A C{str} indicating the header to return. @return: The C{str} header value from the request if it exists, else C{None}. """ value = self.requestHeaders.getRawHeaders(key) if value is not None: return value[-1] def getResponseHeader(self, key): """Get a header from the response. @param key: A C{str} indicating the header to return. @return: The C{str} header value from the response if it exists, else C{None}. """ value = self.responseHeaders.getRawHeaders(key) if value is not None: return value[-1] @property def response(self): """The HTTP response body.""" return self.written.getvalue() def getAllHeaders(self): """ Get all the request headers. This is copied from Twisted's C{twisted.web.http.Request} class. @return: A C{dict} of request header name -> value. """ headers = {} for k, v in self.requestHeaders.getAllRawHeaders(): headers[k.lower()] = v[-1] return headers def notifyFinish(self): """ Return a C{twisted.internet.Deferred} that fires when the request finishes or errors if the client disconnects. Note that this method is needed as resource.py calls it, but we do not need to actually fire the returned deferred for our tests (which cause synchronous exceptions to immediately be returned to the request errback). @return: A C{Deferred} as just described. """ return succeed(None) def isSecure(self): """Is the request secure? @return: a C{bool} that is C{True} if the request is secure. """ # The way we tell if a request is secure is based on a header set # for us by nginx. Our Twisted web service only handles plain http # requests that are forwarded to it by nginx (via haproxy). secureHeader = self.getHeader('X-Forwarded-Protocol') return (secureHeader == 'https')
class T2WRequest(http.Request): """ Used by Tor2webProxy to implement a simple web proxy. """ def __init__(self, channel, queued, reactor=reactor): """ Method overridden to change some part of proxy.Request and of the base http.Request """ self.reactor = reactor self.notifications = [] self.channel = channel self.queued = queued self.requestHeaders = Headers() self.received_cookies = {} self.responseHeaders = Headers() self.cookies = [] # outgoing cookies self.bodyProducer = BodyProducer() self.proxy_d = None self.proxy_response = None self.stream = '' self.header_injected = False # If we should disable the banner, # say that we have already injected it. if config.disable_banner: self.header_injected = True if queued: self.transport = StringTransport() else: self.transport = self.channel.transport self.obj = Tor2webObj() self.var = Storage() self.var['version'] = __version__ self.var['basehost'] = config.basehost self.var['errorcode'] = None self.html = False self.decoderGzip = None self.encoderGzip = None self.pool = pool def _cleanup(self): """ Method overridden to avoid self.content actions. """ if self.producer: log.err(RuntimeError("Producer was not unregistered for %s" % self.uri)) self.unregisterProducer() self.channel.requestDone(self) del self.channel for d in self.notifications: d.callback(None) self.notifications = [] def getRequestHostname(self): """ Function overload to fix ipv6 bug: http://twistedmatrix.com/trac/ticket/6014 """ host = self.getHeader(b'host') if host: if host[0]=='[': return host.split(']',1)[0] + "]" return networkString(host.split(':', 1)[0]) return networkString(self.getHost().host) def forwardData(self, data, end=False): if not self.startedWriting: if self.obj.client_supports_gzip: self.setHeader(b'content-encoding', b'gzip') if data != '' and end: self.setHeader(b'content-length', intToBytes(len(data))) if data != '': try: self.write(data) except Exception: pass def requestReceived(self, command, path, version): """ Method overridden to reduce the function actions """ self.method, self.uri = command, path self.clientproto = version # cache the client and server information, we'll need this later to be # serialized and sent with the request so CGIs will work remotely self.client = self.channel.transport.getPeer() self.host = self.channel.transport.getHost() self.process() def add_banner(self, banner, data): """ Inject tor2web banner inside the returned page """ return str(data.group(1)) + str(banner) @defer.inlineCallbacks def handleFixPart(self, data): if self.obj.server_response_is_gzip: data = self.unzip(data) data = self.stream + data if len(data) >= 1000: data = re_sub(rexp['t2w'], r'https://\2.' + config.basehost, data) forward = data[:-500] if not self.header_injected and forward.find("<body") != -1: banner = yield flattenString(self, templates['banner.tpl']) forward = re.sub(rexp['body'], partial(self.add_banner, banner), forward) self.header_injected = True self.forwardData(self.handleCleartextForwardPart(forward)) self.stream = data[-500:] else: self.stream = data @defer.inlineCallbacks def handleFixEnd(self, data): if self.obj.server_response_is_gzip: data = self.unzip(data, True) data = self.stream + data data = re_sub(rexp['t2w'], r'https://\2.' + config.basehost, data) if not self.header_injected and data.find("<body") != -1: banner = yield flattenString(self, templates['banner.tpl']) data = re.sub(rexp['body'], partial(self.add_banner, banner), data) self.header_injected = True data = self.handleCleartextForwardPart(data, True) self.forwardData(data, True) self.stream = '' try: self.finish() except Exception: pass def handleGzippedForwardPart(self, data, end=False): if not self.obj.client_supports_gzip: data = self.unzip(data, end) return data def handleCleartextForwardPart(self, data, end=False): if self.obj.client_supports_gzip: data = self.zip(data, end) return data def handleForwardPart(self, data): if self.obj.server_response_is_gzip: data = self.handleGzippedForwardPart(data) else: data = self.handleCleartextForwardPart(data) self.forwardData(data) def handleForwardEnd(self, data): if self.obj.server_response_is_gzip: data = self.handleGzippedForwardPart(data, True) else: data = self.handleCleartextForwardPart(data, True) self.forwardData(data, True) try: self.finish() except Exception: pass def contentFinish(self, data): if self.obj.client_supports_gzip: self.setHeader(b'content-encoding', b'gzip') data = self.zip(data, True) self.setHeader(b'content-length', intToBytes(len(data))) self.setHeader(b'cache-control', b'no-cache') if config.blockcrawl: self.setHeader(b'X-Robots-Tag', b'noindex') if self.isSecure(): self.setHeader(b'strict-transport-security', b'max-age=31536000') try: self.write(data) self.finish() except Exception: pass def sendError(self, error=500, errortemplate='error_generic.tpl'): self.setResponseCode(error) self.setHeader(b'content-type', 'text/html') self.var['errorcode'] = error return flattenString(self, templates[errortemplate]).addCallback(self.contentFinish) def handleError(self, failure): if type(failure.value) is SOCKSError: self.setResponseCode(404) self.var['errorcode'] = failure.value.code if failure.value.code in SOCKS_errors: return flattenString(self, templates[SOCKS_errors[failure.value.code]]).addCallback(self.contentFinish) else: return flattenString(self, templates[SOCKS_errors[0x00]]).addCallback(self.contentFinish) else: self.sendError() def unzip(self, data, end=False): data1 = data2 = '' try: if self.decoderGzip is None: self.decoderGzip = zlib.decompressobj(16 + zlib.MAX_WBITS) if data != '': data1 = self.decoderGzip.decompress(data) if end: data2 = self.decoderGzip.flush() except Exception: pass return data1 + data2 def zip(self, data, end=False): data1 = data2 = '' try: if self.encoderGzip is None: self.encoderGzip = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS) if data != '': data1 = self.encoderGzip.compress(data) if end: data2 = self.encoderGzip.flush() except Exception: pass return data1 + data2 def process_request(self, req): """ This function: - "resolves" the address; - alters and sets the proper headers. """ rpc_log(req) self.obj.host_tor = "http://" + self.obj.onion self.obj.host_tor2web = "https://" + self.obj.onion.replace(".onion", "") + "." + config.basehost self.obj.address = "http://" + self.obj.onion + self.obj.uri self.obj.headers = req.headers rpc_log("Headers before fix:") rpc_log(self.obj.headers) self.obj.headers.removeHeader(b'if-modified-since') self.obj.headers.removeHeader(b'if-none-match') self.obj.headers.setRawHeaders(b'host', [self.obj.onion]) self.obj.headers.setRawHeaders(b'connection', [b'keep-alive']) self.obj.headers.setRawHeaders(b'Accept-encoding', [b'gzip, chunked']) self.obj.headers.setRawHeaders(b'x-tor2web', [b'encrypted']) for key, values in self.obj.headers.getAllRawHeaders(): fixed_values = [] for value in values: value = re_sub(rexp['w2t'], r'http://\2.onion', value) fixed_values.append(value) self.obj.headers.setRawHeaders(key, fixed_values) rpc_log("Headers after fix:") rpc_log(self.obj.headers) return True @defer.inlineCallbacks def process(self): request = Storage() request.headers = self.requestHeaders request.host = self.getRequestHostname() request.uri = self.uri content_length = self.getHeader(b'content-length') transfer_encoding = self.getHeader(b'transfer-encoding') staticpath = request.uri staticpath = re.sub('/$', '/index.html', staticpath) staticpath = re.sub('^(/antanistaticmap/)?', '', staticpath) staticpath = re.sub('^/', '', staticpath) resource_is_local = (config.mode != "TRANSLATION" and (request.host == config.basehost or request.host == 'www.' + config.basehost)) or \ isIPAddress(request.host) or \ isIPv6Address(request.host) or \ (config.overriderobotstxt and request.uri == '/robots.txt') or \ request.uri.startswith('/antanistaticmap/') if content_length is not None: self.bodyProducer.length = int(content_length) producer = self.bodyProducer request.headers.removeHeader(b'content-length') elif transfer_encoding is not None: producer = self.bodyProducer request.headers.removeHeader(b'transfer-encoding') else: producer = None if config.mirror is not None: if config.basehost in config.mirror: config.mirror.remove(config.basehost) if len(config.mirror) > 1: self.var['mirror'] = choice(config.mirror) elif len(config.mirror) == 1: self.var['mirror'] = config.mirror[0] # we serve contents only over https if not self.isSecure() and (config.transport != 'HTTP'): self.redirect("https://" + request.host + request.uri) self.finish() defer.returnValue(None) # 0: Request admission control stage # we try to deny some ua/crawlers regardless the request is (valid or not) / (local or not) # we deny EVERY request to known user agents reconized with pattern matching if config.blockcrawl and request.headers.getRawHeaders(b'user-agent') is not None: for ua in blocked_ua_list: if re.match(ua, request.headers.getRawHeaders(b'user-agent')[0].lower()): self.sendError(403, "error_blocked_ua.tpl") defer.returnValue(NOT_DONE_YET) # 1: Client capability assessment stage if request.headers.getRawHeaders(b'accept-encoding') is not None: if re.search('gzip', request.headers.getRawHeaders(b'accept-encoding')[0]): self.obj.client_supports_gzip = True # 2: Content delivery stage # we need to verify if the requested resource is local (/antanistaticmap/*) or remote # because some checks must be done only for remote requests; # in fact local content is always served (css, js, and png in fact are used in errors) if resource_is_local: # the requested resource is local, we deliver it directly try: if staticpath == "dev/null": content = "A" * random.randint(20, 1024) self.setHeader(b'content-type', 'text/plain') defer.returnValue(self.contentFinish(content)) elif staticpath == "stats/yesterday": self.setHeader(b'content-type', 'application/json') content = yield rpc("get_yesterday_stats") defer.returnValue(self.contentFinish(content)) elif staticpath == "notification": ################################################################# # Here we need to parse POST data in x-www-form-urlencoded format ################################################################# content_receiver = BodyReceiver(defer.Deferred()) self.bodyProducer.startProducing(content_receiver) yield self.bodyProducer.finished content = ''.join(content_receiver._data) args = {} ctype = self.requestHeaders.getRawHeaders(b'content-type') if ctype is not None: ctype = ctype[0] if self.method == b"POST" and ctype: key, pdict = parse_header(ctype) if key == b'application/x-www-form-urlencoded': args.update(parse_qs(content, 1)) ################################################################# if 'by' in args and 'url' in args and 'comment' in args: tmp = [] tmp.append("From: Tor2web Node %s.%s <%s>\n" % (config.nodename, config.basehost, config.smtpmail)) tmp.append("To: %s\n" % config.smtpmailto_notifications) tmp.append("Subject: Tor2web Node (IPv4 %s, IPv6 %s): notification for %s\n" % (config.listen_ipv4, config.listen_ipv6, args['url'][0])) tmp.append("Content-Type: text/plain; charset=ISO-8859-1\n") tmp.append("Content-Transfer-Encoding: 8bit\n\n") tmp.append("BY: %s\n" % (args['by'][0])) tmp.append("URL: %s\n" % (args['url'][0])) tmp.append("COMMENT: %s\n" % (args['comment'][0])) message = StringIO(''.join(tmp)) try: sendmail(config.smtpuser, config.smtppass, config.smtpmail, config.smtpmailto_notifications, message, config.smtpdomain, config.smtpport) except Exception: pass self.setHeader(b'content-type', 'text/plain') defer.returnValue(self.contentFinish('')) else: if type(antanistaticmap[staticpath]) == str: filename, ext = os.path.splitext(staticpath) self.setHeader(b'content-type', mimetypes.types_map[ext]) content = antanistaticmap[staticpath] defer.returnValue(self.contentFinish(content)) elif type(antanistaticmap[staticpath]) == PageTemplate: defer.returnValue(flattenString(self, antanistaticmap[staticpath]).addCallback(self.contentFinish)) except Exception: pass self.sendError(404) defer.returnValue(NOT_DONE_YET) else: self.obj.uri = request.uri if not request.host: self.sendError(406, 'error_invalid_hostname.tpl') defer.returnValue(NOT_DONE_YET) if config.mode == "TRANSLATION": self.obj.onion = config.onion else: self.obj.onion = request.host.split(".")[0] + ".onion" rpc_log("detected <onion_url>.tor2web Hostname: %s" % self.obj.onion) if not verify_onion(self.obj.onion): self.sendError(406, 'error_invalid_hostname.tpl') defer.returnValue(NOT_DONE_YET) if config.mode == "ACCESSLIST": if not hashlib.md5(self.obj.onion) in access_list: self.sendError(403, 'error_hs_completely_blocked.tpl') defer.returnValue(NOT_DONE_YET) elif config.mode == "BLACKLIST": if hashlib.md5(self.obj.onion).hexdigest() in access_list: self.sendError(403, 'error_hs_completely_blocked.tpl') defer.returnValue(NOT_DONE_YET) if hashlib.md5(self.obj.onion + self.obj.uri).hexdigest() in access_list: self.sendError(403, 'error_hs_specific_page_blocked.tpl') defer.returnValue(NOT_DONE_YET) # we need to verify if the user is using tor; # on this condition it's better to redirect on the .onion if self.getClientIP() in tor_exits_list: self.redirect("http://" + self.obj.onion + request.uri) try: self.finish() except Exception: pass defer.returnValue(None) # Avoid image hotlinking if request.uri.lower().endswith(('gif','jpg','png')): if request.headers.getRawHeaders(b'referer') is not None and \ not config.basehost in request.headers.getRawHeaders(b'referer')[0].lower(): self.sendError(403) defer.returnValue(NOT_DONE_YET) # the requested resource is remote, we act as proxy self.process_request(request) parsed = urlparse(self.obj.address) self.var['address'] = self.obj.address self.var['onion'] = self.obj.onion.replace(".onion", "") self.var['path'] = parsed[2] if parsed[3] is not None and parsed[3] != '': self.var['path'] += '?' + parsed[3] agent = Agent(reactor, sockhost=config.sockshost, sockport=config.socksport, pool=self.pool) if config.dummyproxy is None: proxy_url = 's' + self.obj.address else: proxy_url = config.dummyproxy + parsed[2] + '?' + parsed[3] self.proxy_d = agent.request(self.method, proxy_url, self.obj.headers, bodyProducer=producer) self.proxy_d.addCallback(self.cbResponse) self.proxy_d.addErrback(self.handleError) defer.returnValue(NOT_DONE_YET) def cbResponse(self, response): self.proxy_response = response if 600 <= int(response.code) <= 699: self.setResponseCode(500) self.var['errorcode'] = int(response.code) - 600 if self.var['errorcode'] in SOCKS_errors: return flattenString(self, templates[SOCKS_errors[self.var['errorcode']]]).addCallback(self.contentFinish) else: return flattenString(self, templates[SOCKS_errors[0x00]]).addCallback(self.contentFinish) self.setResponseCode(response.code) self.processResponseHeaders(response.headers) if response.length is not 0: finished = defer.Deferred() if self.obj.html: response.deliverBody(BodyStreamer(self.handleFixPart, finished)) finished.addCallback(self.handleFixEnd) else: response.deliverBody(BodyStreamer(self.handleForwardPart, finished)) finished.addCallback(self.handleForwardEnd) return finished else: self.contentFinish('') return defer.succeed def handleHeader(self, key, values): keyLower = key.lower() # some headers does not allow multiple occurrences # in case of multiple occurrences we evaluate only the first valueLower = values[0].lower() if keyLower == 'transfer-encoding' and valueLower == 'chunked': return elif keyLower == 'content-encoding' and valueLower == 'gzip': self.obj.server_response_is_gzip = True return elif keyLower == 'content-type' and re.search('text/html', valueLower): self.obj.html = True elif keyLower == 'content-length': self.receivedContentLen = valueLower return elif keyLower == 'cache-control': return if keyLower in 'location': fixed_values = [] for value in values: value = re_sub(rexp['t2w'], r'https://\2.' + config.basehost, value) fixed_values.append(value) values = fixed_values self.responseHeaders.setRawHeaders(key, values) def processResponseHeaders(self, headers): # currently we track only responding hidden services # we don't need to block on the rpc now so no yield is needed rpc("update_stats", str(self.obj.onion.replace(".onion", ""))) for key, values in headers.getAllRawHeaders(): self.handleHeader(key, values) def connectionLost(self, reason): try: if self.proxy_d: self.proxy_d.cancel() except Exception: pass try: if self.proxy_response: self.proxy_response._transport.stopProducing() except Exception: pass try: http.Request.connectionLost(self, reason) except Exception: pass def finish(self): try: http.Request.finish(self) except Exception: pass
class FakeRequest(object): """ I pretend to be an HTTP request, with a handful of required methods used by Twisted resource classes (e.g., write, finish, setResponseCode). """ _fluidDB_reqid = None body = '' def __init__(self, method, d=None, headers=None, hostname=None): """ Initialize. d (if not None) is a deferred that will fire with the body of the response when request.finish is called. @param d: A C{Deferred} instance to fire when the request is finished. @param headers: a C{dict} of headers and values for the request. """ self.method = method self.d = d self.uri = None self.requestHeaders = Headers(headers) self.responseHeaders = Headers() self.args = {'verb': [method]} self.hostname = hostname self.content = StringIO.StringIO() def isSecure(self): """ Mark the request as not being secure (i.e., not HTTPS). The isSecure is used by the FluidDB wsfe to construct the Location header in the response to a POST. @return: A constant, C{False}, seeing as we don't care about the return value, only that we provide the method. """ return False def finish(self): """ Indicate that the request has been fully serviced. Send the response body (as accumulated by self.write) back to the deferred (if any) we were passed in __init__. @return: C{None}. """ if self.d: self.d.callback(self.body) def write(self, data): """ Fake the writing of data back to the client. Instead, we accumuluate it so as to deliver it to our test client when self.finish is called. @param data: A C{str} of response payload data. @return: C{None}. """ self.body += data def getHeader(self, key): """ Get a header from the request. This is copied from Twisted's twisted.web.http.Request class. @param key: A C{str} indicating the header to return. @return: The C{str} header value from the request if it exists, else C{None}. """ value = self.requestHeaders.getRawHeaders(key) if value is not None: return value[-1] def getResponseHeader(self, key): """ Get a header from the response. @param key: A C{str} indicating the header to return. @return: The C{str} header value from the response if it exists, else C{None}. """ value = self.responseHeaders.getRawHeaders(key) if value is not None: return value[-1] def setHeader(self, name, value): """ Set a header for the HTTP response. This is copied from Twisted's twisted.web.http.Request class. @param name: A C{str} indicating the header to set. @param value: A C{str} values for the header. @return: C{None}. """ self.responseHeaders.setRawHeaders(name, [value]) def getAllHeaders(self): """ Get all the request headers. This is copied from Twisted's twisted.web.http.Request class. @return: A C{dict} of request header name -> value. """ headers = {} for k, v in self.requestHeaders.getAllRawHeaders(): headers[k.lower()] = v[-1] return headers def setResponseCode(self, code): """ Set the response status code. @param code: An HTTP status code. @return: C{None}. """ self.status = code def notifyFinish(self): """ Return a C{twisted.internet.Deferred} that fires when the request finishes or errors if the client disconnects. Note that this method is needed as resource.py calls it, but we do not need to actually fire the returned deferred for our tests (which cause synchronous exceptions to immediately be returned to the request errback). @return: C{twisted.internet.Deferred} """ return defer.succeed(None) def getRequestHostname(self): """ Return the hostname that was used for issuing this request. """ return self.hostname
class FakeRequest(object): """ A fake request suitable for use in place of C{twisted.web.http.Request}. @param method: The HTTP method being invoked such as C{GET} or C{POST}. @param uri: The URI of the resource being requested. @param headers: A C{dict} of headers to send in the request. @param args: A C{dict} of arguments that should be appended to the URI in the request. """ content = None def __init__(self, method, uri, headers=None, args=None): self.method = method self.uri = uri self.requestHeaders = Headers(headers) self.responseHeaders = Headers() self.args = {'verb': [method]} if args: self.args.update(args) self._fluidDB_reqid = 'xxx' self.finished = False def finish(self): """ Indicate that all response data has been written to this request. """ self.finished = True def getHeader(self, key): """ Get the value of an HTTP request header. @param key: The name of the HTTP header to retrieve a value for. @return: The value set for the header or C{None} if a value is not available. """ value = self.requestHeaders.getRawHeaders(key) if value is not None: return value[-1] def getResponseHeader(self, key): """ Get the value of an HTTP response header. @param key: The name of the HTTP header to retrieve a value for. @return: The value set for the header or C{None} if a value is not available. """ value = self.responseHeaders.getRawHeaders(key) if value is not None: return value[-1] def setHeader(self, name, value): """ Set an HTTP header to include in the response returned to the client. @param name: The name of the header to set. @param value: The value to set. """ self.responseHeaders.setRawHeaders(name, [value]) def getAllHeaders(self): """ Return all the request headers. @return: A C{dict} of all headers, with their values. """ headers = {} for k, v in self.requestHeaders.getAllRawHeaders(): headers[k.lower()] = v[-1] return headers def setResponseCode(self, code): """ Set the HTTP response code to return to the client. @param code: An HTTP response code as defined in C{twisted.web.http}. """ self.status = code def notifyFinish(self): """ Return a C{twisted.internet.Deferred} that fires when the request finishes or errors if the client disconnects. Note that this method is needed as resource.py calls it, but we do not need to actually fire the returned deferred for our test (which causes a synchronous exception to immediately be returned to the request errback). @return: C{twisted.internet.Deferred} """ return defer.succeed(None)
class FakeRequest(Componentized): """ Implementation of L{inevow.IRequest} which is convenient to use in unit tests. @ivar lastModified: The value passed to L{setLastModified} or C{None} if that method has not been called. @type accumulator: C{str} @ivar accumulator: The bytes written to the response body. @type deferred: L{Deferred} @ivar deferred: The deferred which represents rendering of the response to this request. This is basically an implementation detail of L{NevowRequest}. Application code should probably never use this. @ivar _appRootURL: C{None} or the object passed to L{rememberRootURL}. """ fields = None failure = None context = None redirected_to = None lastModified = None content = "" method = 'GET' code = http.OK deferred = None accumulator = '' _appRootURL = None def __init__(self, headers=None, args=None, avatar=None, uri='/', currentSegments=None, cookies=None, user="", password="", isSecure=False): """ Create a FakeRequest instance. @param headers: dict of request headers @param args: dict of args @param avatar: avatar to pass to the FakeSession instance @param uri: request URI @param currentSegments: list of segments that have "already been located" @param cookies: dict of cookies @param user: username (like in http auth) @param password: password (like in http auth) @param isSecure: whether this request represents an HTTPS url """ Componentized.__init__(self) self.uri = uri if not uri.startswith('/'): raise ValueError('uri must be relative with absolute path') self.path = uri self.prepath = [] postpath = uri.split('?')[0] assert postpath.startswith('/') self.postpath = postpath[1:].split('/') if currentSegments is not None: for seg in currentSegments: assert seg == self.postpath[0] self.prepath.append(self.postpath.pop(0)) else: self.prepath.append('') self.responseHeaders = Headers() self.args = args or {} self.sess = FakeSession(avatar) self.site = FakeSite() self.requestHeaders = Headers() if headers: for k, v in headers.items(): self.requestHeaders.setRawHeaders(k, [v]) if cookies is not None: self.cookies = cookies else: self.cookies = {} self.user = user self.password = password self.secure = isSecure self.deferred = defer.Deferred() def URLPath(self): from nevow import url return url.URL.fromContext(self) def getSession(self): return self.sess def registerProducer(self, producer, streaming): """ Synchronously cause the given producer to produce all of its data. This will not work with push producers. Do not use it with them. """ keepGoing = [None] self.unregisterProducer = keepGoing.pop while keepGoing: producer.resumeProducing() del self.unregisterProducer def v(): def get(self): return self.accumulator return get, v = property(*v()) def write(self, bytes): """ Accumulate the given bytes as part of the response body. @type bytes: C{str} """ self.accumulator += bytes finished = False def finishRequest(self, success): self.finished = True def finish(self): self.deferred.callback('') def getHeader(self, key): return self.requestHeaders.getRawHeaders(key, [None])[0] def setHeader(self, key, val): self.responseHeaders.setRawHeaders(key, [val]) def redirect(self, url): self.redirected_to = url def processingFailed(self, f): self.failure = f def setResponseCode(self, code): self.code = code def setLastModified(self, when): self.lastModified = when def prePathURL(self): """ The absolute URL up until the last handled segment of this request. @rtype: C{str}. """ return 'http://%s/%s' % (self.getHeader('host') or 'localhost', '/'.join(self.prepath)) def getClientIP(self): return '127.0.0.1' def addCookie(self, k, v, expires=None, domain=None, path=None, max_age=None, comment=None, secure=None): """ Set a cookie for use in subsequent requests. """ self.cookies[k] = v def getCookie(self, k): """ Fetch a cookie previously set. """ return self.cookies.get(k) def getUser(self): """ Returns the HTTP auth username. """ return self.user def getPassword(self): """ Returns the HTTP auth password. """ return self.password def getRootURL(self): """ Return the previously remembered URL. """ return self._appRootURL def rememberRootURL(self, url=None): """ For compatibility with appserver.NevowRequest. """ if url is None: raise NotImplementedError( "Default URL remembering logic is not implemented.") self._appRootURL = url def isSecure(self): """ Returns whether this is an HTTPS request or not. """ return self.secure def _warnHeaders(self, old, new): """ Emit a warning related to use of one of the deprecated C{headers} or C{received_headers} attributes. @param old: The name of the deprecated attribute to which the warning pertains. @param new: The name of the preferred attribute which replaces the old attribute. """ warnings.warn( category=DeprecationWarning, message=("nevow.testutil.FakeRequest.%(old)s was deprecated in " "Nevow 0.13.0: Please use nevow.testutil.FakeRequest." "%(new)s instead." % dict(old=old, new=new)), stacklevel=3) @property def headers(self): """ Transform the L{Headers}-style C{responseHeaders} attribute into a deprecated C{dict}-style C{headers} attribute. """ self._warnHeaders("headers", "responseHeaders") return appserver._DictHeaders(self.responseHeaders) @property def received_headers(self): """ Transform the L{Headers}-style C{requestHeaders} attribute into a deprecated C{dict}-style C{received_headers} attribute. """ self._warnHeaders("received_headers", "requestHeaders") return appserver._DictHeaders(self.requestHeaders)
class DummyRequest(object): """ Represents a dummy or fake request. See L{twisted.web.server.Request}. @ivar _finishedDeferreds: L{None} or a C{list} of L{Deferreds} which will be called back with L{None} when C{finish} is called or which will be errbacked if C{processingFailed} is called. @type requestheaders: C{Headers} @ivar requestheaders: A Headers instance that stores values for all request headers. @type responseHeaders: C{Headers} @ivar responseHeaders: A Headers instance that stores values for all response headers. @type responseCode: C{int} @ivar responseCode: The response code which was passed to C{setResponseCode}. @type written: C{list} of C{bytes} @ivar written: The bytes which have been written to the request. """ uri = b'http://dummy/' method = b'GET' client = None def registerProducer(self, prod, s): self.go = 1 while self.go: prod.resumeProducing() def unregisterProducer(self): self.go = 0 def __init__(self, postpath, session=None): self.sitepath = [] self.written = [] self.finished = 0 self.postpath = postpath self.prepath = [] self.session = None self.protoSession = session or Session(0, self) self.args = {} self.requestHeaders = Headers() self.responseHeaders = Headers() self.responseCode = None self._finishedDeferreds = [] self._serverName = b"dummy" self.clientproto = b"HTTP/1.0" def getAllHeaders(self): """ Return dictionary mapping the names of all received headers to the last value received for each. Since this method does not return all header information, C{self.requestHeaders.getAllRawHeaders()} may be preferred. NOTE: This function is a direct copy of C{twisted.web.http.Request.getAllRawHeaders}. """ headers = {} for k, v in self.requestHeaders.getAllRawHeaders(): headers[k.lower()] = v[-1] return headers def getHeader(self, name): """ Retrieve the value of a request header. @type name: C{bytes} @param name: The name of the request header for which to retrieve the value. Header names are compared case-insensitively. @rtype: C{bytes} or L{None} @return: The value of the specified request header. """ return self.requestHeaders.getRawHeaders(name.lower(), [None])[0] def setHeader(self, name, value): """TODO: make this assert on write() if the header is content-length """ self.responseHeaders.addRawHeader(name, value) def getSession(self): if self.session: return self.session assert not self.written, "Session cannot be requested after data has been written." self.session = self.protoSession return self.session def render(self, resource): """ Render the given resource as a response to this request. This implementation only handles a few of the most common behaviors of resources. It can handle a render method that returns a string or C{NOT_DONE_YET}. It doesn't know anything about the semantics of request methods (eg HEAD) nor how to set any particular headers. Basically, it's largely broken, but sufficient for some tests at least. It should B{not} be expanded to do all the same stuff L{Request} does. Instead, L{DummyRequest} should be phased out and L{Request} (or some other real code factored in a different way) used. """ result = resource.render(self) if result is NOT_DONE_YET: return self.write(result) self.finish() def write(self, data): if not isinstance(data, bytes): raise TypeError("write() only accepts bytes") self.written.append(data) def notifyFinish(self): """ Return a L{Deferred} which is called back with L{None} when the request is finished. This will probably only work if you haven't called C{finish} yet. """ finished = Deferred() self._finishedDeferreds.append(finished) return finished def finish(self): """ Record that the request is finished and callback and L{Deferred}s waiting for notification of this. """ self.finished = self.finished + 1 if self._finishedDeferreds is not None: observers = self._finishedDeferreds self._finishedDeferreds = None for obs in observers: obs.callback(None) def processingFailed(self, reason): """ Errback and L{Deferreds} waiting for finish notification. """ if self._finishedDeferreds is not None: observers = self._finishedDeferreds self._finishedDeferreds = None for obs in observers: obs.errback(reason) def addArg(self, name, value): self.args[name] = [value] def setResponseCode(self, code, message=None): """ Set the HTTP status response code, but takes care that this is called before any data is written. """ assert not self.written, "Response code cannot be set after data has been written: %s." % "@@@@".join( self.written) self.responseCode = code self.responseMessage = message def setLastModified(self, when): assert not self.written, "Last-Modified cannot be set after data has been written: %s." % "@@@@".join( self.written) def setETag(self, tag): assert not self.written, "ETag cannot be set after data has been written: %s." % "@@@@".join( self.written) def getClientIP(self): """ Return the IPv4 address of the client which made this request, if there is one, otherwise L{None}. """ if isinstance(self.client, IPv4Address): return self.client.host return None def getRequestHostname(self): """ Get a dummy hostname associated to the HTTP request. @rtype: C{bytes} @returns: a dummy hostname """ return self._serverName def getHost(self): """ Get a dummy transport's host. @rtype: C{IPv4Address} @returns: a dummy transport's host """ return IPv4Address('TCP', '127.0.0.1', 80) def setHost(self, host, port, ssl=0): """ Change the host and port the request thinks it's using. @type host: C{bytes} @param host: The value to which to change the host header. @type ssl: C{bool} @param ssl: A flag which, if C{True}, indicates that the request is considered secure (if C{True}, L{isSecure} will return C{True}). """ self._forceSSL = ssl # set first so isSecure will work if self.isSecure(): default = 443 else: default = 80 if port == default: hostHeader = host else: hostHeader = host + b":" + intToBytes(port) self.requestHeaders.addRawHeader(b"host", hostHeader) def getClient(self): """ Get the client's IP address, if it has one. @return: The same value as C{getClientIP}. @rtype: L{bytes} """ return self.getClientIP() def redirect(self, url): """ Utility function that does a redirect. The request should have finish() called after this. """ self.setResponseCode(FOUND) self.setHeader(b"location", url)
class FakeRequest(object): """ A fake C{twisted.web.http.Request} implementation, suitable for use in tests. """ def __init__(self, args=None, postpath=None, prepath=None, path=None, uri=None, method='GET', headers=None, body=''): self.args = {} if args is None else args self.written = StringIO() self.finished = False self.code = None self.method = method self.path = path self.postpath = postpath self.prepath = prepath self.requestHeaders = Headers({}) if headers is None else headers self.responseHeaders = Headers({}) self.uri = uri self._fluidDB_reqid = requestId() self.content = StringIO(body) def write(self, content): """Write data as a result of an HTTP request. @param content: A C{str} containing the bytes to send as part of the response body. """ if not isinstance(content, str): raise RuntimeError('Only strings can be written.') self.written.write(content) def finish(self): """Indicate that all response data has been written to this request.""" if self.code is None: self.code = 200 self.finished = True def setResponseCode(self, code): """Set the HTTP response code. @param code: An C{int} HTTP status code. """ self.code = code def setHeader(self, key, value): """Set an HTTP response header. @param key: The name of the response header. @param value: The value for the response header. """ self.responseHeaders.setRawHeaders(key, [value]) def getHeader(self, key): """ Get a header from the request. This is copied from Twisted's C{twisted.web.http.Request} class. @param key: A C{str} indicating the header to return. @return: The C{str} header value from the request if it exists, else C{None}. """ value = self.requestHeaders.getRawHeaders(key) if value is not None: return value[-1] def getResponseHeader(self, key): """Get a header from the response. @param key: A C{str} indicating the header to return. @return: The C{str} header value from the response if it exists, else C{None}. """ value = self.responseHeaders.getRawHeaders(key) if value is not None: return value[-1] @property def response(self): """The HTTP response body.""" return self.written.getvalue() def getAllHeaders(self): """ Get all the request headers. This is copied from Twisted's C{twisted.web.http.Request} class. @return: A C{dict} of request header name -> value. """ headers = {} for k, v in self.requestHeaders.getAllRawHeaders(): headers[k.lower()] = v[-1] return headers def notifyFinish(self): """ Return a C{twisted.internet.Deferred} that fires when the request finishes or errors if the client disconnects. Note that this method is needed as resource.py calls it, but we do not need to actually fire the returned deferred for our tests (which cause synchronous exceptions to immediately be returned to the request errback). @return: A C{Deferred} as just described. """ return succeed(None) def isSecure(self): """Is the request secure? @return: a C{bool} that is C{True} if the request is secure. """ # The way we tell if a request is secure is based on a header set # for us by nginx. Our Twisted web service only handles plain http # requests that are forwarded to it by nginx (via haproxy). secureHeader = self.getHeader('X-Forwarded-Protocol') return (secureHeader == 'https')