def test_copy(self):
"""
L{Headers.copy} creates a new independent copy of an existing
L{Headers} instance, allowing future modifications without impacts
between the copies.
"""
h = Headers()
h.setRawHeaders("test\u00E1", ["foo\u2603"])
i = h.copy()
# The copy contains the same value as the original
self.assertEqual(i.getRawHeaders("test\u00E1"), ["foo\u2603"])
self.assertEqual(i.getRawHeaders(b"test\xe1"), [b"foo\xe2\x98\x83"])
# Add a header to the original
h.addRawHeader("test\u00E1", "bar")
# Verify that the copy has not changed
self.assertEqual(i.getRawHeaders("test\u00E1"), ["foo\u2603"])
self.assertEqual(i.getRawHeaders(b"test\xe1"), [b"foo\xe2\x98\x83"])
# Add a header to the copy
i.addRawHeader("test\u00E1", b"baz")
# Verify that the orignal does not have it
self.assertEqual(h.getRawHeaders("test\u00E1"), ["foo\u2603", "bar"])
self.assertEqual(h.getRawHeaders(b"test\xe1"),
[b"foo\xe2\x98\x83", b"bar"])
def test_copy(self):
"""
L{Headers.copy} creates a new independent copy of an existing
L{Headers} instance, allowing future modifications without impacts
between the copies.
"""
h = Headers()
h.setRawHeaders(u'test\u00E1', [u'foo\u2603'])
i = h.copy()
# The copy contains the same value as the original
self.assertEqual(i.getRawHeaders(u'test\u00E1'), [u'foo\u2603'])
self.assertEqual(i.getRawHeaders(b'test\xe1'), [b'foo\xe2\x98\x83'])
# Add a header to the original
h.addRawHeader(u'test\u00E1', u'bar')
# Verify that the copy has not changed
self.assertEqual(i.getRawHeaders(u'test\u00E1'), [u'foo\u2603'])
self.assertEqual(i.getRawHeaders(b'test\xe1'), [b'foo\xe2\x98\x83'])
# Add a header to the copy
i.addRawHeader(u'test\u00E1', b'baz')
# Verify that the orignal does not have it
self.assertEqual(
h.getRawHeaders(u'test\u00E1'), [u'foo\u2603', u'bar'])
self.assertEqual(
h.getRawHeaders(b'test\xe1'), [b'foo\xe2\x98\x83', b'bar'])
def test_addRawHeader(self):
"""
L{Headers.addRawHeader} adds a new value for a given header.
"""
h = Headers()
h.addRawHeader("test", "lemur")
self.assertEqual(h.getRawHeaders("test"), ["lemur"])
h.addRawHeader("test", "panda")
self.assertEqual(h.getRawHeaders("test"), ["lemur", "panda"])
def test_addRawHeader(self):
"""
L{Headers.addRawHeader} adds a new value for a given header.
"""
h = Headers()
h.addRawHeader(b"test", b"lemur")
self.assertEqual(h.getRawHeaders(b"test"), [b"lemur"])
h.addRawHeader(b"test", b"panda")
self.assertEqual(h.getRawHeaders(b"test"), [b"lemur", b"panda"])
def test_initializer(self):
"""
The header values passed to L{Headers.__init__} can be retrieved via
L{Headers.getRawHeaders}. If a L{bytes} argument is given, it returns
L{bytes} values, and if a L{unicode} argument is given, it returns
L{unicode} values. Both are the same header value, just encoded or
decoded.
"""
h = Headers({"Foo": ["bar"]})
self.assertEqual(h.getRawHeaders(b"foo"), [b"bar"])
self.assertEqual(h.getRawHeaders("foo"), ["bar"])
def test_getRawHeadersWithDefaultMatchingValue(self):
"""
If the object passed as the value list to L{Headers.setRawHeaders}
is later passed as a default to L{Headers.getRawHeaders}, the
result nevertheless contains decoded values.
"""
h = Headers()
default = [b"value"]
h.setRawHeaders(b"key", default)
self.assertIsInstance(h.getRawHeaders("key", default)[0], str)
self.assertEqual(h.getRawHeaders("key", default), ["value"])
def test_getRawHeaders(self):
"""
L{Headers.getRawHeaders} returns the values which have been set for a
given header.
"""
h = Headers()
h.setRawHeaders("test\u00E1", ["lemur"])
self.assertEqual(h.getRawHeaders("test\u00E1"), ["lemur"])
self.assertEqual(h.getRawHeaders("Test\u00E1"), ["lemur"])
self.assertEqual(h.getRawHeaders(b"test\xe1"), [b"lemur"])
self.assertEqual(h.getRawHeaders(b"Test\xe1"), [b"lemur"])
def test_getRawHeadersWithDefaultMatchingValue(self):
"""
If the object passed as the value list to L{Headers.setRawHeaders}
is later passed as a default to L{Headers.getRawHeaders}, the
result nevertheless contains encoded values.
"""
h = Headers()
default = [u"value"]
h.setRawHeaders(b"key", default)
self.assertIsInstance(h.getRawHeaders(b"key", default)[0], bytes)
self.assertEqual(h.getRawHeaders(b"key", default), [b"value"])
def test_getRawHeaders(self):
"""
L{Headers.getRawHeaders} returns the values which have been set for a
given header.
"""
h = Headers()
h.setRawHeaders(u"test\u00E1", [u"lemur"])
self.assertEqual(h.getRawHeaders(u"test\u00E1"), [u"lemur"])
self.assertEqual(h.getRawHeaders(u"Test\u00E1"), [u"lemur"])
self.assertEqual(h.getRawHeaders(b"test\xe1"), [b"lemur"])
self.assertEqual(h.getRawHeaders(b"Test\xe1"), [b"lemur"])
def test_getRawHeadersDefaultValue(self):
"""
L{Headers.getRawHeaders} returns the specified default value when no
header is found.
"""
h = Headers()
default = object()
self.assertIdentical(h.getRawHeaders(u"test", default), default)
self.assertIdentical(h.getRawHeaders(u"test", None), None)
self.assertEqual(h.getRawHeaders(u"test", [None]), [None])
self.assertEqual(
h.getRawHeaders(u"test", [u"\N{SNOWMAN}"]),
[u"\N{SNOWMAN}"],
)
def test_getRawHeadersDefaultValue(self):
"""
L{Headers.getRawHeaders} returns the specified default value when no
header is found.
"""
h = Headers()
default = object()
self.assertIdentical(h.getRawHeaders("test", default), default)
self.assertIdentical(h.getRawHeaders("test", None), None)
self.assertEqual(h.getRawHeaders("test", [None]), [None])
self.assertEqual(
h.getRawHeaders("test", ["\N{SNOWMAN}"]),
["\N{SNOWMAN}"],
)
def test_setRawHeaders(self):
"""
L{Headers.setRawHeaders} sets the header values for the given
header name to the sequence of strings, encoded.
"""
rawValue = [u"value1", u"value2"]
rawEncodedValue = [b"value1", b"value2"]
h = Headers()
h.setRawHeaders("test", rawValue)
self.assertTrue(h.hasHeader(b"test"))
self.assertTrue(h.hasHeader(b"Test"))
self.assertTrue(h.hasHeader("test"))
self.assertTrue(h.hasHeader("Test"))
self.assertEqual(h.getRawHeaders("test"), rawValue)
self.assertEqual(h.getRawHeaders(b"test"), rawEncodedValue)
def test_setRawHeaders(self):
"""
L{Headers.setRawHeaders} sets the header values for the given
header name to the sequence of strings, encoded.
"""
rawValue = ["value1", "value2"]
rawEncodedValue = [b"value1", b"value2"]
h = Headers()
h.setRawHeaders("test", rawValue)
self.assertTrue(h.hasHeader(b"test"))
self.assertTrue(h.hasHeader(b"Test"))
self.assertTrue(h.hasHeader("test"))
self.assertTrue(h.hasHeader("Test"))
self.assertEqual(h.getRawHeaders("test"), rawValue)
self.assertEqual(h.getRawHeaders(b"test"), rawEncodedValue)
def test_initializer(self):
"""
The header values passed to L{Headers.__init__} can be retrieved via
L{Headers.getRawHeaders}.
"""
h = Headers({'Foo': ['bar']})
self.assertEqual(h.getRawHeaders('foo'), ['bar'])
def test_initializer(self):
"""
The header values passed to L{Headers.__init__} can be retrieved via
L{Headers.getRawHeaders}.
"""
h = Headers({b"Foo": [b"bar"]})
self.assertEqual(h.getRawHeaders(b"foo"), [b"bar"])
def test_initializer(self):
"""
The header values passed to L{Headers.__init__} can be retrieved via
L{Headers.getRawHeaders}.
"""
h = Headers({b"Foo": [b"bar"]})
self.assertEqual(h.getRawHeaders(b"foo"), [b"bar"])
def test_initializer(self):
"""
The header values passed to L{Headers.__init__} can be retrieved via
L{Headers.getRawHeaders}.
"""
h = Headers({b'Foo': [b'bar']})
self.assertEqual(h.getRawHeaders(b'foo'), [b'bar'])
def check_content_type_is(headers: Headers,
expected_content_type: str) -> None:
"""
Check that a set of HTTP headers have a Content-Type header, and that it
is the expected value..
Args:
headers: headers to check
Raises:
RequestSendFailed: if the Content-Type header is missing or doesn't match
"""
content_type_headers = headers.getRawHeaders(b"Content-Type")
if content_type_headers is None:
raise RequestSendFailed(
RuntimeError("No Content-Type header received from remote server"),
can_retry=False,
)
c_type = content_type_headers[0].decode("ascii") # only the first header
val, options = cgi.parse_header(c_type)
if val != expected_content_type:
raise RequestSendFailed(
RuntimeError(
f"Remote server sent Content-Type header of '{c_type}', not '{expected_content_type}'",
),
can_retry=False,
)
def _cache_period_from_headers(
headers: Headers,
time_now: Callable[[], float] = time.time) -> Optional[float]:
cache_controls = _parse_cache_control(headers)
if b"no-store" in cache_controls:
return 0
if b"max-age" in cache_controls:
max_age = cache_controls[b"max-age"]
if max_age:
try:
return int(max_age)
except ValueError:
pass
expires = headers.getRawHeaders(b"expires")
if expires is not None:
try:
expires_date = stringToDatetime(expires[-1])
return expires_date - time_now()
except ValueError:
# RFC7234 says 'A cache recipient MUST interpret invalid date formats,
# especially the value "0", as representing a time in the past (i.e.,
# "already expired").
return 0
return None
def check_content_type_is_json(headers: Headers) -> None:
"""
Check that a set of HTTP headers have a Content-Type header, and that it
is application/json.
Args:
headers: headers to check
Raises:
RequestSendFailed: if the Content-Type header is missing or isn't JSON
"""
c_type = headers.getRawHeaders(b"Content-Type")
if c_type is None:
raise RequestSendFailed(
RuntimeError("No Content-Type header received from remote server"),
can_retry=False,
)
c_type = c_type[0].decode("ascii") # only the first header
val, options = cgi.parse_header(c_type)
if val != "application/json":
raise RequestSendFailed(
RuntimeError(
"Remote server sent Content-Type header of '%s', not 'application/json'"
% c_type, ),
can_retry=False,
)
def test_getRawHeadersDefaultValue(self):
"""
L{Headers.getRawHeaders} returns the specified default value when no
header is found.
"""
h = Headers()
default = object()
self.assertIdentical(h.getRawHeaders(b"test", default), default)
def test_getRawHeadersDefaultValue(self):
"""
L{Headers.getRawHeaders} returns the specified default value when no
header is found.
"""
h = Headers()
default = object()
self.assertIdentical(h.getRawHeaders("test", default), default)
def test_rawHeadersValueEncoding(self):
"""
Passing L{unicode} to L{Headers.setRawHeaders} will encode the name as
ISO-8859-1 and values as UTF-8.
"""
h = Headers()
h.setRawHeaders(u"\u00E1", [u"\u2603", b"foo"])
self.assertTrue(h.hasHeader(b"\xe1"))
self.assertEqual(h.getRawHeaders(b"\xe1"), [b'\xe2\x98\x83', b'foo'])
def get_content_type(headers: Headers) -> Optional[str]:
"""
Get the content type from the HTTP ``Content-Type`` header.
Returns ``None`` if no content-type was set.
"""
values = headers.getRawHeaders("content-type") or [None]
content_type = parse_options_header(values[0])[0] or None
return content_type
def test_rawHeadersValueEncoding(self):
"""
Passing L{unicode} to L{Headers.setRawHeaders} will encode the name as
ISO-8859-1 and values as UTF-8.
"""
h = Headers()
h.setRawHeaders("\u00E1", ["\u2603", b"foo"])
self.assertTrue(h.hasHeader(b"\xe1"))
self.assertEqual(h.getRawHeaders(b"\xe1"), [b"\xe2\x98\x83", b"foo"])
class _FakeUrllib2Request(object):
"""
A fake C{urllib2.Request} object for C{cookielib} to work with.
@see: U{http://docs.python.org/library/urllib2.html#request-objects}
@type uri: C{str}
@ivar uri: Request URI.
@type headers: L{twisted.web.http_headers.Headers}
@ivar headers: Request headers.
@type type: C{str}
@ivar type: The scheme of the URI.
@type host: C{str}
@ivar host: The host[:port] of the URI.
@since: 11.1
"""
def __init__(self, uri):
self.uri = uri
self.headers = Headers()
self.type, rest = splittype(self.uri)
self.host, rest = splithost(rest)
def has_header(self, header):
return self.headers.hasHeader(header)
def add_unredirected_header(self, name, value):
self.headers.addRawHeader(name, value)
def get_full_url(self):
return self.uri
def get_header(self, name, default=None):
headers = self.headers.getRawHeaders(name, default)
if headers is not None:
return headers[0]
return None
def get_host(self):
return self.host
def get_type(self):
return self.type
def is_unverifiable(self):
# In theory this shouldn't be hardcoded.
return False
def test_getRawHeaders(self):
"""
L{Headers.getRawHeaders} returns the values which have been set for a
given header.
"""
h = Headers()
h.setRawHeaders("test", ["lemur"])
self.assertEqual(h.getRawHeaders("test"), ["lemur"])
self.assertEqual(h.getRawHeaders("Test"), ["lemur"])
def _parse_cache_control(headers: Headers) -> Dict[bytes, Optional[bytes]]:
cache_controls = {}
for hdr in headers.getRawHeaders(b"cache-control", []):
for directive in hdr.split(b","):
splits = [x.strip() for x in directive.split(b"=", 1)]
k = splits[0].lower()
v = splits[1] if len(splits) > 1 else None
cache_controls[k] = v
return cache_controls
def test_initializer(self):
"""
The header values passed to L{Headers.__init__} can be retrieved via
L{Headers.getRawHeaders}. If a L{bytes} argument is given, it returns
L{bytes} values, and if a L{unicode} argument is given, it returns
L{unicode} values. Both are the same header value, just encoded or
decoded.
"""
h = Headers({u'Foo': [u'bar']})
self.assertEqual(h.getRawHeaders(b'foo'), [b'bar'])
self.assertEqual(h.getRawHeaders(u'foo'), [u'bar'])
def request(self, method, uri, headers, bodyProducer):
if headers is None:
headers = Headers()
else:
headers = headers.copy()
contentType = headers.getRawHeaders('content-type', [""])[0]
date = headers.getRawHeaders('date',
[""])[0] or self._generateRequestDate(uri)
headers.setRawHeaders('date', [date])
uri_origin_form = URI.fromBytes(uri).originForm
contentMD5 = headers.getRawHeaders('content-md5', [""])[0]
if not contentMD5 and bodyProducer is not None:
r = getattr(self.agent, '_reactor') or reactor
bodyConsumer = StringConsumer(callLater=r.callLater)
yield bodyProducer.startProducing(bodyConsumer)
body = bodyConsumer.value()
bodyProducer = StringBodyProducer(body)
if body:
contentMD5 = binascii.b2a_base64(
hashlib.md5(body).digest()).strip()
headers.addRawHeader('content-md5', contentMD5)
sts = "\n".join([
method, contentType or "", contentMD5, date or "", uri_origin_form
])
mac = hmac.new(self.secretKey, sts, digestmod=hashlib.sha1).digest()
encodedMAC = binascii.b2a_base64(mac).strip()
auth_header = "AuthHMAC {0}:{1}".format(self.accessKey, encodedMAC)
headers.addRawHeader('authorization', auth_header)
d = yield self.agent.request(method, uri, headers, bodyProducer)
self._handleResponseDate(uri, d)
defer.returnValue(d)
def test_copy(self):
"""
L{Headers.copy} creates a new independant copy of an existing
L{Headers} instance, allowing future modifications without impacts
between the copies.
"""
h = Headers()
h.setRawHeaders('test', ['foo'])
i = h.copy()
self.assertEquals(i.getRawHeaders('test'), ['foo'])
h.addRawHeader('test', 'bar')
self.assertEquals(i.getRawHeaders('test'), ['foo'])
i.addRawHeader('test', 'baz')
self.assertEquals(h.getRawHeaders('test'), ['foo', 'bar'])
class _FakeUrllib2Request(object):
"""
A fake C{urllib2.Request} object for C{cookielib} to work with.
@see: U{http://docs.python.org/library/urllib2.html#request-objects}
@type uri: C{str}
@ivar uri: Request URI.
@type headers: L{twisted.web.http_headers.Headers}
@ivar headers: Request headers.
@type type: C{str}
@ivar type: The scheme of the URI.
@type host: C{str}
@ivar host: The host[:port] of the URI.
@since: 11.1
"""
def __init__(self, uri):
self.uri = uri
self.headers = Headers()
self.type, rest = splittype(self.uri)
self.host, rest = splithost(rest)
def has_header(self, header):
return self.headers.hasHeader(header)
def add_unredirected_header(self, name, value):
self.headers.addRawHeader(name, value)
def get_full_url(self):
return self.uri
def get_header(self, name, default=None):
headers = self.headers.getRawHeaders(name, default)
if headers is not None:
return headers[0]
return None
def get_host(self):
return self.host
def get_type(self):
return self.type
def is_unverifiable(self):
# In theory this shouldn't be hardcoded.
return False
def test_copy(self):
"""
L{Headers.copy} creates a new independant copy of an existing
L{Headers} instance, allowing future modifications without impacts
between the copies.
"""
h = Headers()
h.setRawHeaders(b"test", [b"foo"])
i = h.copy()
self.assertEqual(i.getRawHeaders(b"test"), [b"foo"])
h.addRawHeader(b"test", b"bar")
self.assertEqual(i.getRawHeaders(b"test"), [b"foo"])
i.addRawHeader(b"test", b"baz")
self.assertEqual(h.getRawHeaders(b"test"), [b"foo", b"bar"])
def test_copy(self):
"""
L{Headers.copy} creates a new independant copy of an existing
L{Headers} instance, allowing future modifications without impacts
between the copies.
"""
h = Headers()
h.setRawHeaders(b'test', [b'foo'])
i = h.copy()
self.assertEqual(i.getRawHeaders(b'test'), [b'foo'])
h.addRawHeader(b'test', b'bar')
self.assertEqual(i.getRawHeaders(b'test'), [b'foo'])
i.addRawHeader(b'test', b'baz')
self.assertEqual(h.getRawHeaders(b'test'), [b'foo', b'bar'])
def test_copy(self):
"""
L{Headers.copy} creates a new independent copy of an existing
L{Headers} instance, allowing future modifications without impacts
between the copies.
"""
h = Headers()
h.setRawHeaders(b"test", [b"foo"])
i = h.copy()
self.assertEqual(i.getRawHeaders(b"test"), [b"foo"])
h.addRawHeader(b"test", b"bar")
self.assertEqual(i.getRawHeaders(b"test"), [b"foo"])
i.addRawHeader(b"test", b"baz")
self.assertEqual(h.getRawHeaders(b"test"), [b"foo", b"bar"])
def request(self, method, uri, headers, bodyProducer):
if headers is None:
headers = Headers()
else:
headers = headers.copy()
contentType = headers.getRawHeaders('content-type', [""])[0]
date = headers.getRawHeaders('date', [""])[0] or self._generateRequestDate(uri)
headers.setRawHeaders('date', [date])
uri_origin_form = URI.fromBytes(uri).originForm
contentMD5 = headers.getRawHeaders('content-md5', [""])[0]
if not contentMD5 and bodyProducer is not None:
r = getattr(self.agent, '_reactor') or reactor
bodyConsumer = StringConsumer(callLater=r.callLater)
yield bodyProducer.startProducing(bodyConsumer)
body = bodyConsumer.value()
bodyProducer = StringBodyProducer(body)
if body:
contentMD5 = binascii.b2a_base64(hashlib.md5(body).digest()).strip()
headers.addRawHeader('content-md5', contentMD5)
sts = "\n".join([method, contentType or "", contentMD5, date or "", uri_origin_form])
mac = hmac.new(self.secretKey, sts, digestmod=hashlib.sha1).digest()
encodedMAC = binascii.b2a_base64(mac).strip()
auth_header = "AuthHMAC {0}:{1}".format(self.accessKey, encodedMAC)
headers.addRawHeader('authorization', auth_header)
d = yield self.agent.request(method, uri, headers, bodyProducer)
self._handleResponseDate(uri, d)
defer.returnValue(d)
class FakeRequest(object):
"""
A fake request suitable for use in place of C{twisted.web.http.Request}.
@param method: The HTTP method being invoked such as C{GET} or C{POST}.
"""
def __init__(self, method):
self.method = method
self.requestHeaders = Headers()
self.responseHeaders = Headers()
self._fluidDB_reqid = 'xxx'
self.finished = False
def finish(self):
"""
Indicate that all response data has been written to this request.
"""
self.finished = True
def getResponseHeader(self, name):
"""
Get the value of an HTTP response header.
@param name: The name of the HTTP header to retrieve a value for.
@return: The value set for the header or C{None} if a value is not
available.
"""
value = self.responseHeaders.getRawHeaders(name)
if value is not None:
return value[-1]
def setHeader(self, name, value):
"""
Set an HTTP header to include in the response returned to the client.
@param name: The name of the header to set.
@param value: The value to set.
"""
self.responseHeaders.setRawHeaders(name, [value])
def setResponseCode(self, code):
"""
Set the HTTP response code to return to the client.
@param code: An HTTP response code as defined in C{twisted.web.http}.
"""
self.status = code
def test_nameEncoding(self):
"""
Passing L{unicode} to any function that takes a header name will encode
said header name as ISO-8859-1.
"""
h = Headers()
# We set it using a Unicode string.
h.setRawHeaders(u"\u00E1", [b"foo"])
# It's encoded to the ISO-8859-1 value, which we can use to access it
self.assertTrue(h.hasHeader(b"\xe1"))
self.assertEqual(h.getRawHeaders(b"\xe1"), [b'foo'])
# We can still access it using the Unicode string..
self.assertTrue(h.hasHeader(u"\u00E1"))
def test_nameEncoding(self):
"""
Passing L{unicode} to any function that takes a header name will encode
said header name as ISO-8859-1.
"""
h = Headers()
# We set it using a Unicode string.
h.setRawHeaders("\u00E1", [b"foo"])
# It's encoded to the ISO-8859-1 value, which we can use to access it
self.assertTrue(h.hasHeader(b"\xe1"))
self.assertEqual(h.getRawHeaders(b"\xe1"), [b"foo"])
# We can still access it using the Unicode string..
self.assertTrue(h.hasHeader("\u00E1"))
class FakeRequest(object):
"""
A fake request suitable for use in place of C{twisted.web.http.Request}.
@param method: The HTTP method being invoked such as C{GET} or C{POST}.
"""
def __init__(self, method):
self.method = method
self.requestHeaders = Headers()
self.responseHeaders = Headers()
self._fluidDB_reqid = 'xxx'
self.finished = False
def finish(self):
"""
Indicate that all response data has been written to this request.
"""
self.finished = True
def getResponseHeader(self, name):
"""
Get the value of an HTTP response header.
@param name: The name of the HTTP header to retrieve a value for.
@return: The value set for the header or C{None} if a value is not
available.
"""
value = self.responseHeaders.getRawHeaders(name)
if value is not None:
return value[-1]
def setHeader(self, name, value):
"""
Set an HTTP header to include in the response returned to the client.
@param name: The name of the header to set.
@param value: The value to set.
"""
self.responseHeaders.setRawHeaders(name, [value])
def setResponseCode(self, code):
"""
Set the HTTP response code to return to the client.
@param code: An HTTP response code as defined in C{twisted.web.http}.
"""
self.status = code
class _FakeUrllib2Request(object):
"""
A fake C{urllib2.Request} object for C{cookielib} to work with.
@type uri: C{str}
@ivar uri: Request URI.
@type headers: L{twisted.web.http_headers.Headers}
@ivar headers: Request headers.
@since: 11.1
"""
def __init__(self, uri):
self.uri = uri
self.headers = Headers()
def has_header(self, header):
return self.headers.hasHeader(header)
def add_unredirected_header(self, name, value):
self.headers.addRawHeader(name, value)
def get_full_url(self):
return self.uri
def get_header(self, name, default=None):
headers = self.headers.getRawHeaders(name, default)
if headers is not None:
return headers[0]
return None
def is_unverifiable(self):
# In theory this shouldn't be hardcoded.
return False
class FakeRequest(Componentized):
"""
Implementation of L{inevow.IRequest} which is convenient to use in unit
tests.
@ivar lastModified: The value passed to L{setLastModified} or C{None} if
that method has not been called.
@type accumulator: C{str}
@ivar accumulator: The bytes written to the response body.
@type deferred: L{Deferred}
@ivar deferred: The deferred which represents rendering of the response
to this request. This is basically an implementation detail of
L{NevowRequest}. Application code should probably never use this.
@ivar _appRootURL: C{None} or the object passed to L{rememberRootURL}.
"""
implements(inevow.IRequest)
fields = None
failure = None
context = None
redirected_to = None
lastModified = None
content = ""
method = 'GET'
code = http.OK
deferred = None
accumulator = ''
_appRootURL = None
def __init__(self, headers=None, args=None, avatar=None,
uri='/', currentSegments=None, cookies=None,
user="", password="", isSecure=False):
"""
Create a FakeRequest instance.
@param headers: dict of request headers
@param args: dict of args
@param avatar: avatar to pass to the FakeSession instance
@param uri: request URI
@param currentSegments: list of segments that have "already been located"
@param cookies: dict of cookies
@param user: username (like in http auth)
@param password: password (like in http auth)
@param isSecure: whether this request represents an HTTPS url
"""
Componentized.__init__(self)
self.uri = uri
if not uri.startswith('/'):
raise ValueError('uri must be relative with absolute path')
self.path = uri
self.prepath = []
postpath = uri.split('?')[0]
assert postpath.startswith('/')
self.postpath = postpath[1:].split('/')
if currentSegments is not None:
for seg in currentSegments:
assert seg == self.postpath[0]
self.prepath.append(self.postpath.pop(0))
else:
self.prepath.append('')
self.responseHeaders = Headers()
self.args = args or {}
self.sess = FakeSession(avatar)
self.site = FakeSite()
self.requestHeaders = Headers()
if headers:
for k, v in headers.iteritems():
self.requestHeaders.setRawHeaders(k, [v])
if cookies is not None:
self.cookies = cookies
else:
self.cookies = {}
self.user = user
self.password = password
self.secure = isSecure
self.deferred = defer.Deferred()
def URLPath(self):
from nevow import url
return url.URL.fromContext(self)
def getSession(self):
return self.sess
def registerProducer(self, producer, streaming):
"""
Synchronously cause the given producer to produce all of its data.
This will not work with push producers. Do not use it with them.
"""
keepGoing = [None]
self.unregisterProducer = keepGoing.pop
while keepGoing:
producer.resumeProducing()
del self.unregisterProducer
def v():
def get(self):
return self.accumulator
return get,
v = property(*v())
def write(self, bytes):
"""
Accumulate the given bytes as part of the response body.
@type bytes: C{str}
"""
self.accumulator += bytes
finished = False
def finishRequest(self, success):
self.finished = True
def finish(self):
self.deferred.callback('')
def getHeader(self, key):
return self.requestHeaders.getRawHeaders(key, [None])[0]
def setHeader(self, key, val):
self.responseHeaders.setRawHeaders(key, [val])
def redirect(self, url):
self.redirected_to = url
def processingFailed(self, f):
self.failure = f
def setResponseCode(self, code):
self.code = code
def setLastModified(self, when):
self.lastModified = when
def prePathURL(self):
"""
The absolute URL up until the last handled segment of this request.
@rtype: C{str}.
"""
return 'http://%s/%s' % (self.getHeader('host') or 'localhost',
'/'.join(self.prepath))
def getClientIP(self):
return '127.0.0.1'
def addCookie(self, k, v, expires=None, domain=None, path=None, max_age=None, comment=None, secure=None):
"""
Set a cookie for use in subsequent requests.
"""
self.cookies[k] = v
def getCookie(self, k):
"""
Fetch a cookie previously set.
"""
return self.cookies.get(k)
def getUser(self):
"""
Returns the HTTP auth username.
"""
return self.user
def getPassword(self):
"""
Returns the HTTP auth password.
"""
return self.password
def getRootURL(self):
"""
Return the previously remembered URL.
"""
return self._appRootURL
def rememberRootURL(self, url=None):
"""
For compatibility with appserver.NevowRequest.
"""
if url is None:
raise NotImplementedError(
"Default URL remembering logic is not implemented.")
self._appRootURL = url
def isSecure(self):
"""
Returns whether this is an HTTPS request or not.
"""
return self.secure
def _warnHeaders(self, old, new):
"""
Emit a warning related to use of one of the deprecated C{headers} or
C{received_headers} attributes.
@param old: The name of the deprecated attribute to which the warning
pertains.
@param new: The name of the preferred attribute which replaces the old
attribute.
"""
warnings.warn(
category=DeprecationWarning,
message=(
"nevow.testutil.FakeRequest.%(old)s was deprecated in "
"Nevow 0.13.0: Please use nevow.testutil.FakeRequest."
"%(new)s instead." % dict(old=old, new=new)),
stacklevel=3)
@property
def headers(self):
"""
Transform the L{Headers}-style C{responseHeaders} attribute into a
deprecated C{dict}-style C{headers} attribute.
"""
self._warnHeaders("headers", "responseHeaders")
return appserver._DictHeaders(self.responseHeaders)
@property
def received_headers(self):
"""
Transform the L{Headers}-style C{requestHeaders} attribute into a
deprecated C{dict}-style C{received_headers} attribute.
"""
self._warnHeaders("received_headers", "requestHeaders")
return appserver._DictHeaders(self.requestHeaders)
class FakeRequest(object):
"""
I pretend to be an HTTP request, with a handful of required methods
used by Twisted resource classes (e.g., write, finish,
setResponseCode).
"""
_fluidDB_reqid = None
body = ''
def __init__(self, method, d=None, headers=None, hostname=None):
"""
Initialize. d (if not None) is a deferred that will fire with the
body of the response when request.finish is called.
@param d: A C{Deferred} instance to fire when the request is finished.
@param headers: a C{dict} of headers and values for the request.
"""
self.method = method
self.d = d
self.uri = None
self.requestHeaders = Headers(headers)
self.responseHeaders = Headers()
self.args = {'verb': [method]}
self.hostname = hostname
self.content = StringIO.StringIO()
def isSecure(self):
"""
Mark the request as not being secure (i.e., not HTTPS). The
isSecure is used by the FluidDB wsfe to construct the Location
header in the response to a POST.
@return: A constant, C{False}, seeing as we don't care about
the return value, only that we provide the method.
"""
return False
def finish(self):
"""
Indicate that the request has been fully serviced. Send the
response body (as accumulated by self.write) back to the deferred
(if any) we were passed in __init__.
@return: C{None}.
"""
if self.d:
self.d.callback(self.body)
def write(self, data):
"""
Fake the writing of data back to the client. Instead, we
accumuluate it so as to deliver it to our test client when
self.finish is called.
@param data: A C{str} of response payload data.
@return: C{None}.
"""
self.body += data
def getHeader(self, key):
"""
Get a header from the request. This is copied from Twisted's
twisted.web.http.Request class.
@param key: A C{str} indicating the header to return.
@return: The C{str} header value from the request if it exists,
else C{None}.
"""
value = self.requestHeaders.getRawHeaders(key)
if value is not None:
return value[-1]
def getResponseHeader(self, key):
"""
Get a header from the response.
@param key: A C{str} indicating the header to return.
@return: The C{str} header value from the response if it exists,
else C{None}.
"""
value = self.responseHeaders.getRawHeaders(key)
if value is not None:
return value[-1]
def setHeader(self, name, value):
"""
Set a header for the HTTP response. This is copied from Twisted's
twisted.web.http.Request class.
@param name: A C{str} indicating the header to set.
@param value: A C{str} values for the header.
@return: C{None}.
"""
self.responseHeaders.setRawHeaders(name, [value])
def getAllHeaders(self):
"""
Get all the request headers. This is copied from Twisted's
twisted.web.http.Request class.
@return: A C{dict} of request header name -> value.
"""
headers = {}
for k, v in self.requestHeaders.getAllRawHeaders():
headers[k.lower()] = v[-1]
return headers
def setResponseCode(self, code):
"""
Set the response status code.
@param code: An HTTP status code.
@return: C{None}.
"""
self.status = code
def notifyFinish(self):
"""
Return a C{twisted.internet.Deferred} that fires when the request
finishes or errors if the client disconnects. Note that this method
is needed as resource.py calls it, but we do not need to actually
fire the returned deferred for our tests (which cause synchronous
exceptions to immediately be returned to the request errback).
@return: C{twisted.internet.Deferred}
"""
return defer.succeed(None)
def getRequestHostname(self):
"""
Return the hostname that was used for issuing this request.
"""
return self.hostname
class T2WRequest(http.Request):
"""
Used by Tor2webProxy to implement a simple web proxy.
"""
def __init__(self, channel, queued, reactor=reactor):
"""
Method overridden to change some part of proxy.Request and of the base http.Request
"""
self.reactor = reactor
self.notifications = []
self.channel = channel
self.queued = queued
self.requestHeaders = Headers()
self.received_cookies = {}
self.responseHeaders = Headers()
self.cookies = [] # outgoing cookies
self.bodyProducer = BodyProducer()
self.proxy_d = None
self.proxy_response = None
self.stream = ''
self.header_injected = False
if queued:
self.transport = StringTransport()
else:
self.transport = self.channel.transport
self.obj = Tor2webObj()
self.var = Storage()
self.var['version'] = VERSION
self.var['basehost'] = config.basehost
self.var['errorcode'] = None
self.html = False
self.decoderGzip = None
self.encoderGzip = None
self.pool = pool
def _cleanup(self):
"""
Method overridden to avoid self.content actions.
"""
if self.producer:
log.err(
RuntimeError("Producer was not unregistered for %s" %
self.uri))
self.unregisterProducer()
self.channel.requestDone(self)
del self.channel
for d in self.notifications:
d.callback(None)
self.notifications = []
def getRequestHostname(self):
"""
Function overload to fix ipv6 bug:
http://twistedmatrix.com/trac/ticket/6014
"""
host = self.getHeader(b'host')
if host:
if host[0] == '[':
return host.split(']', 1)[0] + "]"
return networkString(host.split(':', 1)[0])
return networkString(self.getHost().host)
def forwardData(self, data, end=False):
if not self.startedWriting:
if self.obj.client_supports_gzip:
self.setHeader(b'content-encoding', b'gzip')
if data != '' and end:
self.setHeader(b'content-length', intToBytes(len(data)))
if data != '':
try:
self.write(data)
except:
pass
def requestReceived(self, command, path, version):
"""
Method overridden to reduce the function actions
"""
self.method, self.uri = command, path
self.clientproto = version
# cache the client and server information, we'll need this later to be
# serialized and sent with the request so CGIs will work remotely
self.client = self.channel.transport.getPeer()
self.host = self.channel.transport.getHost()
self.process()
def add_banner(self, banner, data):
"""
Inject tor2web banner inside the returned page
"""
return str(data.group(1)) + str(banner)
@defer.inlineCallbacks
def handleFixPart(self, data):
if self.obj.server_response_is_gzip:
data = self.unzip(data)
data = self.stream + data
if len(data) >= 1000:
if not self.header_injected and data.find("<body") != -1:
banner = yield flattenString(self, templates['banner.tpl'])
data = re.sub(rexp['body'], partial(self.add_banner, banner),
data)
self.header_injected = True
data = re_sub(rexp['t2w'], r'https://\2.' + config.basehost, data)
self.forwardData(self.handleCleartextForwardPart(data[:-500]))
self.stream = data[-500:]
else:
self.stream = data
@defer.inlineCallbacks
def handleFixEnd(self, data):
if self.obj.server_response_is_gzip:
data = self.unzip(data, True)
data = self.stream + data
if not self.header_injected and data.find("<body") != -1:
banner = yield flattenString(self, templates['banner.tpl'])
data = re.sub(rexp['body'], partial(self.add_banner, banner), data)
self.header_injected = True
data = re_sub(rexp['t2w'], r'https://\2.' + config.basehost, data)
data = self.handleCleartextForwardPart(data, True)
self.forwardData(data, True)
self.stream = ''
try:
self.finish()
except:
pass
def handleGzippedForwardPart(self, data, end=False):
if not self.obj.client_supports_gzip:
data = self.unzip(data, end)
return data
def handleCleartextForwardPart(self, data, end=False):
if self.obj.client_supports_gzip:
data = self.zip(data, end)
return data
def handleForwardPart(self, data):
if self.obj.server_response_is_gzip:
data = self.handleGzippedForwardPart(data)
else:
data = self.handleCleartextForwardPart(data)
self.forwardData(data)
def handleForwardEnd(self, data):
if self.obj.server_response_is_gzip:
data = self.handleGzippedForwardPart(data, True)
else:
data = self.handleCleartextForwardPart(data, True)
self.forwardData(data, True)
try:
self.finish()
except:
pass
def contentFinish(self, data):
if self.obj.client_supports_gzip:
self.setHeader(b'content-encoding', b'gzip')
data = self.zip(data, True)
self.setHeader(b'content-length', intToBytes(len(data)))
try:
self.write(data)
self.finish()
except:
pass
def sendError(self, error=500, errortemplate='error_generic.tpl'):
self.setResponseCode(error)
self.var['errorcode'] = error
return flattenString(self, templates[errortemplate]).addCallback(
self.contentFinish)
def handleError(self, failure):
if type(failure.value) is SOCKSError:
self.setResponseCode(404)
self.var['errorcode'] = failure.value.code
if failure.value.code in SOCKS_errors:
return flattenString(
self,
templates[SOCKS_errors[failure.value.code]]).addCallback(
self.contentFinish)
else:
return flattenString(
self, templates[SOCKS_errors[0x00]]).addCallback(
self.contentFinish)
else:
self.sendError()
def unzip(self, data, end=False):
data1 = data2 = ''
try:
if self.decoderGzip == None:
self.decoderGzip = zlib.decompressobj(16 + zlib.MAX_WBITS)
if data != '':
data1 = self.decoderGzip.decompress(data)
if end:
data2 = self.decoderGzip.flush()
except:
pass
return data1 + data2
def zip(self, data, end=False):
data1 = data2 = ''
try:
if self.encoderGzip == None:
self.encoderGzip = zlib.compressobj(6, zlib.DEFLATED,
16 + zlib.MAX_WBITS)
if data != '':
data1 = self.encoderGzip.compress(data)
if end:
data2 = self.encoderGzip.flush()
except:
pass
return data1 + data2
@defer.inlineCallbacks
def process(self):
content = ""
request = Storage()
request.headers = self.requestHeaders
request.host = self.getRequestHostname()
request.uri = self.uri
content_length = self.getHeader(b'content-length')
transfer_encoding = self.getHeader(b'transfer-encoding')
staticpath = request.uri
staticpath = re.sub('\/$', '/index.html', staticpath)
staticpath = re.sub('^(/antanistaticmap/)?', '', staticpath)
staticpath = re.sub('^/', '', staticpath)
resource_is_local = isIPAddress(request.host) or \
isIPv6Address(request.host) or \
request.uri == '/robots.txt' or \
request.uri.startswith('/antanistaticmap/')
if content_length is not None:
self.bodyProducer.length = int(content_length)
producer = self.bodyProducer
request.headers.removeHeader(b'content-length')
elif transfer_encoding is not None:
producer = self.bodyProducer
request.headers.removeHeader(b'transfer-encoding')
else:
producer = None
if config.mirror is not None:
if config.basehost in config.mirror:
config.mirror.remove(config.basehost)
self.var['mirror'] = choice(config.mirror)
# we serve contents only over https
if not self.isSecure():
self.redirect("https://" + request.host + request.uri)
self.finish()
return
# 0: Request admission control stage
# we try to deny some ua/crawlers regardless the request is (valid or not) / (local or not)
# we deny EVERY request to known user agents reconized with pattern matching
if config.blockcrawl and request.headers.getRawHeaders(
b'user-agent') != None:
for ua in t2w.blocked_ua:
check = request.headers.getRawHeaders(b'user-agent')[0].lower()
if re.match(ua, check):
self.sendError(403, "error_blocked_ua.tpl")
defer.returnValue(NOT_DONE_YET)
# 1: Client capability assessment stage
if request.headers.getRawHeaders(b'accept-encoding') != None:
if re.search('gzip',
request.headers.getRawHeaders(b'accept-encoding')[0]):
self.obj.client_supports_gzip = True
# 2: Content delivery stage
# we need to verify if the requested resource is local (/antanistaticmap/*) or remote
# because some checks must be done only for remote requests;
# in fact local content is always served (css, js, and png in fact are used in errors)
if resource_is_local:
# the requested resource is local, we deliver it directly
try:
if staticpath == "notification":
#################################################################
# Here we need to parse POST data in x-www-form-urlencoded format
#################################################################
content_receiver = BodyReceiver(defer.Deferred())
self.bodyProducer.startProducing(content_receiver)
yield self.bodyProducer.finished
content = ''.join(content_receiver._data)
args = {}
ctype = self.requestHeaders.getRawHeaders(b'content-type')
if ctype is not None:
ctype = ctype[0]
if self.method == b"POST" and ctype:
mfd = b'multipart/form-data'
key, pdict = parse_header(ctype)
if key == b'application/x-www-form-urlencoded':
args.update(parse_qs(content, 1))
#################################################################
if 'by' in args and 'url' in args and 'comment' in args:
tmp = []
tmp.append("From: Tor2web Node %s.%s <%s>\n" %
(config.nodename, config.basehost,
config.smtpmail))
tmp.append("To: %s\n" %
(config.smtpmailto_notifications))
tmp.append(
"Subject: Tor2web Node (IPv4 %s, IPv6 %s): notification for %s\n"
% (config.listen_ipv4, config.listen_ipv6,
args['url'][0]))
tmp.append(
"Content-Type: text/plain; charset=ISO-8859-1\n")
tmp.append("Content-Transfer-Encoding: 8bit\n\n")
tmp.append("BY: %s\n" % (args['by'][0]))
tmp.append("URL: %s\n" % (args['url'][0]))
tmp.append("COMMENT: %s\n" % (args['comment'][0]))
message = StringIO(''.join(tmp))
try:
sendmail(config.smtpuser, config.smtppass,
config.smtpmail,
config.smtpmailto_notifications, message,
config.smtpdomain, config.smtpport)
except:
pass
else:
if type(antanistaticmap[staticpath]) == str:
filename, ext = os.path.splitext(staticpath)
self.setHeader(b'content-type',
mimetypes.types_map[ext])
content = antanistaticmap[staticpath]
defer.returnValue(self.contentFinish(content))
elif type(antanistaticmap[staticpath]) == PageTemplate:
defer.returnValue(
flattenString(
self, antanistaticmap[staticpath]).addCallback(
self.contentFinish))
except:
pass
self.sendError(404)
defer.returnValue(NOT_DONE_YET)
else:
self.obj.uri = request.uri
if not request.host:
self.sendError(406, 'error_invalid_hostname.tpl')
defer.returnValue(NOT_DONE_YET)
if config.mode == "TRANSLATION":
self.obj.onion = config.onion
else:
self.obj.onion = request.host.split(".")[0] + ".onion"
log.msg("detected <onion_url>.tor2web Hostname: %s" %
self.obj.onion)
if not verify_onion(self.obj.onion):
self.sendError(406, 'error_invalid_hostname.tpl')
defer.returnValue(NOT_DONE_YET)
if config.mode == "ACCESSLIST":
if self.obj.onion not in t2w.accesslist:
self.sendError(403, 'error_hs_completely_blocked.tpl')
defer.returnValue(NOT_DONE_YET)
elif config.mode == "BLACKLIST":
if hashlib.md5(
self.obj.onion).hexdigest() in t2w.accesslist:
self.sendError(403, 'error_hs_completely_blocked.tpl')
defer.returnValue(NOT_DONE_YET)
if hashlib.md5(self.obj.onion +
self.obj.uri).hexdigest() in t2w.accesslist:
self.sendError(403,
'error_hs_specific_page_blocked.tpl')
defer.returnValue(NOT_DONE_YET)
# we need to verify if the user is using tor;
# on this condition it's better to redirect on the .onion
if self.getClientIP() in t2w.TorExitNodes:
self.redirect("http://" + self.obj.onion + request.uri)
try:
self.finish()
except:
pass
return
# Avoid image hotlinking
if request.uri.lower().endswith(('gif', 'jpg', 'png')):
if request.headers.getRawHeaders(
b'referer'
) != None and not config.basehost in request.headers.getRawHeaders(
b'referer')[0].lower():
self.sendError(403)
defer.returnValue(NOT_DONE_YET)
# the requested resource is remote, we act as proxy
t2w.process_request(self.obj, request)
parsed = urlparse(self.obj.address)
self.var['address'] = self.obj.address
self.var['onion'] = self.obj.onion.replace(".onion", "")
self.var['path'] = parsed[2] + '?' + parsed[3]
agent = Agent(reactor,
sockhost=config.sockshost,
sockport=config.socksport,
pool=self.pool)
self.proxy_d = agent.request(self.method,
's' + self.obj.address,
self.obj.headers,
bodyProducer=producer)
self.proxy_d.addCallback(self.cbResponse)
self.proxy_d.addErrback(self.handleError)
defer.returnValue(NOT_DONE_YET)
def cbResponse(self, response):
self.proxy_response = response
if int(response.code) >= 600 and int(response.code) <= 699:
self.setResponseCode(500)
self.var['errorcode'] = int(response.code) - 600
if self.var['errorcode'] in SOCKS_errors:
return flattenString(
self, templates[SOCKS_errors[
self.var['errorcode']]]).addCallback(
self.contentFinish)
else:
return flattenString(
self, templates[SOCKS_errors[0x00]]).addCallback(
self.contentFinish)
self.setResponseCode(response.code)
self.processResponseHeaders(response.headers)
if (response.length is not 0):
finished = defer.Deferred()
if self.obj.html:
response.deliverBody(BodyStreamer(self.handleFixPart,
finished))
finished.addCallback(self.handleFixEnd)
else:
response.deliverBody(
BodyStreamer(self.handleForwardPart, finished))
finished.addCallback(self.handleForwardEnd)
return finished
else:
self.contentFinish('')
return defer.succeed
def handleHeader(self, key, values):
keyLower = key.lower()
# some headers does not allow multiple occurrences
# in case of multiple occurrences we evaluate only the first
valueLower = values[0].lower()
if keyLower == 'transfer-encoding' and valueLower == 'chunked':
return
elif keyLower == 'content-encoding' and valueLower == 'gzip':
self.obj.server_response_is_gzip = True
return
elif keyLower == 'content-type' and re.search('text/html', valueLower):
self.obj.html = True
elif keyLower == 'content-length':
self.receivedContentLen = valueLower
return
elif keyLower == 'cache-control':
return
if keyLower in ('location'):
fixed_values = []
for value in values:
value = re_sub(rexp['t2w'], r'https://\2.' + config.basehost,
value)
fixed_values.append(value)
values = fixed_values
self.responseHeaders.setRawHeaders(key, values)
def handleEndHeaders(self):
self.setHeader(b'cache-control', b'no-cache')
self.setHeader(b'strict-transport-security', b'max-age=31536000')
def processResponseHeaders(self, headers):
for key, values in headers.getAllRawHeaders():
self.handleHeader(key, values)
self.handleEndHeaders()
def connectionLost(self, reason):
try:
if self.proxy_d:
self.proxy_d.cancel()
except:
pass
try:
if self.proxy_response:
self.proxy_response._transport.stopProducing()
except:
pass
http.Request.connectionLost(self, reason)
class DummyRequest(object):
"""
Represents a dummy or fake request. See L{twisted.web.server.Request}.
@ivar _finishedDeferreds: C{None} or a C{list} of L{Deferreds} which will
be called back with C{None} when C{finish} is called or which will be
errbacked if C{processingFailed} is called.
@type requestheaders: C{Headers}
@ivar requestheaders: A Headers instance that stores values for all request
headers.
@type responseHeaders: C{Headers}
@ivar responseHeaders: A Headers instance that stores values for all
response headers.
@type responseCode: C{int}
@ivar responseCode: The response code which was passed to
C{setResponseCode}.
@type written: C{list} of C{bytes}
@ivar written: The bytes which have been written to the request.
"""
uri = b'http://dummy/'
method = b'GET'
client = None
def registerProducer(self, prod,s):
self.go = 1
while self.go:
prod.resumeProducing()
def unregisterProducer(self):
self.go = 0
def __init__(self, postpath, session=None):
self.sitepath = []
self.written = []
self.finished = 0
self.postpath = postpath
self.prepath = []
self.session = None
self.protoSession = session or Session(0, self)
self.args = {}
self.requestHeaders = Headers()
self.responseHeaders = Headers()
self.responseCode = None
self._finishedDeferreds = []
self._serverName = b"dummy"
self.clientproto = b"HTTP/1.0"
def getAllHeaders(self):
"""
Return dictionary mapping the names of all received headers to the last
value received for each.
Since this method does not return all header information,
C{self.requestHeaders.getAllRawHeaders()} may be preferred.
NOTE: This function is a direct copy of
C{twisted.web.http.Request.getAllRawHeaders}.
"""
headers = {}
for k, v in self.requestHeaders.getAllRawHeaders():
headers[k.lower()] = v[-1]
return headers
def getHeader(self, name):
"""
Retrieve the value of a request header.
@type name: C{bytes}
@param name: The name of the request header for which to retrieve the
value. Header names are compared case-insensitively.
@rtype: C{bytes} or L{NoneType}
@return: The value of the specified request header.
"""
return self.requestHeaders.getRawHeaders(name.lower(), [None])[0]
def setHeader(self, name, value):
"""TODO: make this assert on write() if the header is content-length
"""
self.responseHeaders.addRawHeader(name, value)
def getSession(self):
if self.session:
return self.session
assert not self.written, "Session cannot be requested after data has been written."
self.session = self.protoSession
return self.session
def render(self, resource):
"""
Render the given resource as a response to this request.
This implementation only handles a few of the most common behaviors of
resources. It can handle a render method that returns a string or
C{NOT_DONE_YET}. It doesn't know anything about the semantics of
request methods (eg HEAD) nor how to set any particular headers.
Basically, it's largely broken, but sufficient for some tests at least.
It should B{not} be expanded to do all the same stuff L{Request} does.
Instead, L{DummyRequest} should be phased out and L{Request} (or some
other real code factored in a different way) used.
"""
result = resource.render(self)
if result is NOT_DONE_YET:
return
self.write(result)
self.finish()
def write(self, data):
if not isinstance(data, bytes):
raise TypeError("write() only accepts bytes")
self.written.append(data)
def notifyFinish(self):
"""
Return a L{Deferred} which is called back with C{None} when the request
is finished. This will probably only work if you haven't called
C{finish} yet.
"""
finished = Deferred()
self._finishedDeferreds.append(finished)
return finished
def finish(self):
"""
Record that the request is finished and callback and L{Deferred}s
waiting for notification of this.
"""
self.finished = self.finished + 1
if self._finishedDeferreds is not None:
observers = self._finishedDeferreds
self._finishedDeferreds = None
for obs in observers:
obs.callback(None)
def processingFailed(self, reason):
"""
Errback and L{Deferreds} waiting for finish notification.
"""
if self._finishedDeferreds is not None:
observers = self._finishedDeferreds
self._finishedDeferreds = None
for obs in observers:
obs.errback(reason)
def addArg(self, name, value):
self.args[name] = [value]
def setResponseCode(self, code, message=None):
"""
Set the HTTP status response code, but takes care that this is called
before any data is written.
"""
assert not self.written, "Response code cannot be set after data has been written: %s." % "@@@@".join(self.written)
self.responseCode = code
self.responseMessage = message
def setLastModified(self, when):
assert not self.written, "Last-Modified cannot be set after data has been written: %s." % "@@@@".join(self.written)
def setETag(self, tag):
assert not self.written, "ETag cannot be set after data has been written: %s." % "@@@@".join(self.written)
def getClientIP(self):
"""
Return the IPv4 address of the client which made this request, if there
is one, otherwise C{None}.
"""
if isinstance(self.client, IPv4Address):
return self.client.host
return None
def getRequestHostname(self):
"""
Get a dummy hostname associated to the HTTP request.
@rtype: C{bytes}
@returns: a dummy hostname
"""
return self._serverName
def getHost(self):
"""
Get a dummy transport's host.
@rtype: C{IPv4Address}
@returns: a dummy transport's host
"""
return IPv4Address('TCP', '127.0.0.1', 80)
def setHost(self, host, port, ssl=0):
"""
Change the host and port the request thinks it's using.
@type host: C{bytes}
@param host: The value to which to change the host header.
@type ssl: C{bool}
@param ssl: A flag which, if C{True}, indicates that the request is
considered secure (if C{True}, L{isSecure} will return C{True}).
"""
self._forceSSL = ssl # set first so isSecure will work
if self.isSecure():
default = 443
else:
default = 80
if port == default:
hostHeader = host
else:
hostHeader = host + b":" + intToBytes(port)
self.requestHeaders.addRawHeader(b"host", hostHeader)
def getClient(self):
"""
Get the client's IP address, if it has one.
@return: The same value as C{getClientIP}.
@rtype: L{bytes}
"""
return self.getClientIP()
def redirect(self, url):
"""
Utility function that does a redirect.
The request should have finish() called after this.
"""
self.setResponseCode(FOUND)
self.setHeader(b"location", url)
class FakeRequest(object):
"""
A fake C{twisted.web.http.Request} implementation, suitable for use in
tests.
"""
def __init__(self,
args=None,
postpath=None,
prepath=None,
path=None,
uri=None,
method='GET',
headers=None,
body=''):
self.args = {} if args is None else args
self.written = StringIO()
self.finished = False
self.code = None
self.method = method
self.path = path
self.postpath = postpath
self.prepath = prepath
self.requestHeaders = Headers({}) if headers is None else headers
self.responseHeaders = Headers({})
self.uri = uri
self._fluidDB_reqid = requestId()
self.content = StringIO(body)
def write(self, content):
"""Write data as a result of an HTTP request.
@param content: A C{str} containing the bytes to send as part of the
response body.
"""
if not isinstance(content, str):
raise RuntimeError('Only strings can be written.')
self.written.write(content)
def finish(self):
"""Indicate that all response data has been written to this request."""
if self.code is None:
self.code = 200
self.finished = True
def setResponseCode(self, code):
"""Set the HTTP response code.
@param code: An C{int} HTTP status code.
"""
self.code = code
def setHeader(self, key, value):
"""Set an HTTP response header.
@param key: The name of the response header.
@param value: The value for the response header.
"""
self.responseHeaders.setRawHeaders(key, [value])
def getHeader(self, key):
"""
Get a header from the request. This is copied from Twisted's
C{twisted.web.http.Request} class.
@param key: A C{str} indicating the header to return.
@return: The C{str} header value from the request if it exists,
else C{None}.
"""
value = self.requestHeaders.getRawHeaders(key)
if value is not None:
return value[-1]
def getResponseHeader(self, key):
"""Get a header from the response.
@param key: A C{str} indicating the header to return.
@return: The C{str} header value from the response if it exists,
else C{None}.
"""
value = self.responseHeaders.getRawHeaders(key)
if value is not None:
return value[-1]
@property
def response(self):
"""The HTTP response body."""
return self.written.getvalue()
def getAllHeaders(self):
"""
Get all the request headers. This is copied from Twisted's
C{twisted.web.http.Request} class.
@return: A C{dict} of request header name -> value.
"""
headers = {}
for k, v in self.requestHeaders.getAllRawHeaders():
headers[k.lower()] = v[-1]
return headers
def notifyFinish(self):
"""
Return a C{twisted.internet.Deferred} that fires when the request
finishes or errors if the client disconnects. Note that this method
is needed as resource.py calls it, but we do not need to actually
fire the returned deferred for our tests (which cause synchronous
exceptions to immediately be returned to the request errback).
@return: A C{Deferred} as just described.
"""
return succeed(None)
def isSecure(self):
"""Is the request secure?
@return: a C{bool} that is C{True} if the request is secure.
"""
# The way we tell if a request is secure is based on a header set
# for us by nginx. Our Twisted web service only handles plain http
# requests that are forwarded to it by nginx (via haproxy).
secureHeader = self.getHeader('X-Forwarded-Protocol')
return (secureHeader == 'https')
class T2WRequest(http.Request):
"""
Used by Tor2webProxy to implement a simple web proxy.
"""
def __init__(self, channel, queued, reactor=reactor):
"""
Method overridden to change some part of proxy.Request and of the base http.Request
"""
self.reactor = reactor
self.notifications = []
self.channel = channel
self.queued = queued
self.requestHeaders = Headers()
self.received_cookies = {}
self.responseHeaders = Headers()
self.cookies = [] # outgoing cookies
self.bodyProducer = BodyProducer()
self.proxy_d = None
self.proxy_response = None
self.stream = ''
self.header_injected = False
# If we should disable the banner,
# say that we have already injected it.
if config.disable_banner:
self.header_injected = True
if queued:
self.transport = StringTransport()
else:
self.transport = self.channel.transport
self.obj = Tor2webObj()
self.var = Storage()
self.var['version'] = __version__
self.var['basehost'] = config.basehost
self.var['errorcode'] = None
self.html = False
self.decoderGzip = None
self.encoderGzip = None
self.pool = pool
def _cleanup(self):
"""
Method overridden to avoid self.content actions.
"""
if self.producer:
log.err(RuntimeError("Producer was not unregistered for %s" % self.uri))
self.unregisterProducer()
self.channel.requestDone(self)
del self.channel
for d in self.notifications:
d.callback(None)
self.notifications = []
def getRequestHostname(self):
"""
Function overload to fix ipv6 bug:
http://twistedmatrix.com/trac/ticket/6014
"""
host = self.getHeader(b'host')
if host:
if host[0]=='[':
return host.split(']',1)[0] + "]"
return networkString(host.split(':', 1)[0])
return networkString(self.getHost().host)
def forwardData(self, data, end=False):
if not self.startedWriting:
if self.obj.client_supports_gzip:
self.setHeader(b'content-encoding', b'gzip')
if data != '' and end:
self.setHeader(b'content-length', intToBytes(len(data)))
if data != '':
try:
self.write(data)
except Exception:
pass
def requestReceived(self, command, path, version):
"""
Method overridden to reduce the function actions
"""
self.method, self.uri = command, path
self.clientproto = version
# cache the client and server information, we'll need this later to be
# serialized and sent with the request so CGIs will work remotely
self.client = self.channel.transport.getPeer()
self.host = self.channel.transport.getHost()
self.process()
def add_banner(self, banner, data):
"""
Inject tor2web banner inside the returned page
"""
return str(data.group(1)) + str(banner)
@defer.inlineCallbacks
def handleFixPart(self, data):
if self.obj.server_response_is_gzip:
data = self.unzip(data)
data = self.stream + data
if len(data) >= 1000:
data = re_sub(rexp['t2w'], r'https://\2.' + config.basehost, data)
forward = data[:-500]
if not self.header_injected and forward.find("<body") != -1:
banner = yield flattenString(self, templates['banner.tpl'])
forward = re.sub(rexp['body'], partial(self.add_banner, banner), forward)
self.header_injected = True
self.forwardData(self.handleCleartextForwardPart(forward))
self.stream = data[-500:]
else:
self.stream = data
@defer.inlineCallbacks
def handleFixEnd(self, data):
if self.obj.server_response_is_gzip:
data = self.unzip(data, True)
data = self.stream + data
data = re_sub(rexp['t2w'], r'https://\2.' + config.basehost, data)
if not self.header_injected and data.find("<body") != -1:
banner = yield flattenString(self, templates['banner.tpl'])
data = re.sub(rexp['body'], partial(self.add_banner, banner), data)
self.header_injected = True
data = self.handleCleartextForwardPart(data, True)
self.forwardData(data, True)
self.stream = ''
try:
self.finish()
except Exception:
pass
def handleGzippedForwardPart(self, data, end=False):
if not self.obj.client_supports_gzip:
data = self.unzip(data, end)
return data
def handleCleartextForwardPart(self, data, end=False):
if self.obj.client_supports_gzip:
data = self.zip(data, end)
return data
def handleForwardPart(self, data):
if self.obj.server_response_is_gzip:
data = self.handleGzippedForwardPart(data)
else:
data = self.handleCleartextForwardPart(data)
self.forwardData(data)
def handleForwardEnd(self, data):
if self.obj.server_response_is_gzip:
data = self.handleGzippedForwardPart(data, True)
else:
data = self.handleCleartextForwardPart(data, True)
self.forwardData(data, True)
try:
self.finish()
except Exception:
pass
def contentFinish(self, data):
if self.obj.client_supports_gzip:
self.setHeader(b'content-encoding', b'gzip')
data = self.zip(data, True)
self.setHeader(b'content-length', intToBytes(len(data)))
self.setHeader(b'cache-control', b'no-cache')
if config.blockcrawl:
self.setHeader(b'X-Robots-Tag', b'noindex')
if self.isSecure():
self.setHeader(b'strict-transport-security', b'max-age=31536000')
try:
self.write(data)
self.finish()
except Exception:
pass
def sendError(self, error=500, errortemplate='error_generic.tpl'):
self.setResponseCode(error)
self.setHeader(b'content-type', 'text/html')
self.var['errorcode'] = error
return flattenString(self, templates[errortemplate]).addCallback(self.contentFinish)
def handleError(self, failure):
if type(failure.value) is SOCKSError:
self.setResponseCode(404)
self.var['errorcode'] = failure.value.code
if failure.value.code in SOCKS_errors:
return flattenString(self, templates[SOCKS_errors[failure.value.code]]).addCallback(self.contentFinish)
else:
return flattenString(self, templates[SOCKS_errors[0x00]]).addCallback(self.contentFinish)
else:
self.sendError()
def unzip(self, data, end=False):
data1 = data2 = ''
try:
if self.decoderGzip is None:
self.decoderGzip = zlib.decompressobj(16 + zlib.MAX_WBITS)
if data != '':
data1 = self.decoderGzip.decompress(data)
if end:
data2 = self.decoderGzip.flush()
except Exception:
pass
return data1 + data2
def zip(self, data, end=False):
data1 = data2 = ''
try:
if self.encoderGzip is None:
self.encoderGzip = zlib.compressobj(6, zlib.DEFLATED, 16 + zlib.MAX_WBITS)
if data != '':
data1 = self.encoderGzip.compress(data)
if end:
data2 = self.encoderGzip.flush()
except Exception:
pass
return data1 + data2
def process_request(self, req):
"""
This function:
- "resolves" the address;
- alters and sets the proper headers.
"""
rpc_log(req)
self.obj.host_tor = "http://" + self.obj.onion
self.obj.host_tor2web = "https://" + self.obj.onion.replace(".onion", "") + "." + config.basehost
self.obj.address = "http://" + self.obj.onion + self.obj.uri
self.obj.headers = req.headers
rpc_log("Headers before fix:")
rpc_log(self.obj.headers)
self.obj.headers.removeHeader(b'if-modified-since')
self.obj.headers.removeHeader(b'if-none-match')
self.obj.headers.setRawHeaders(b'host', [self.obj.onion])
self.obj.headers.setRawHeaders(b'connection', [b'keep-alive'])
self.obj.headers.setRawHeaders(b'Accept-encoding', [b'gzip, chunked'])
self.obj.headers.setRawHeaders(b'x-tor2web', [b'encrypted'])
for key, values in self.obj.headers.getAllRawHeaders():
fixed_values = []
for value in values:
value = re_sub(rexp['w2t'], r'http://\2.onion', value)
fixed_values.append(value)
self.obj.headers.setRawHeaders(key, fixed_values)
rpc_log("Headers after fix:")
rpc_log(self.obj.headers)
return True
@defer.inlineCallbacks
def process(self):
request = Storage()
request.headers = self.requestHeaders
request.host = self.getRequestHostname()
request.uri = self.uri
content_length = self.getHeader(b'content-length')
transfer_encoding = self.getHeader(b'transfer-encoding')
staticpath = request.uri
staticpath = re.sub('/$', '/index.html', staticpath)
staticpath = re.sub('^(/antanistaticmap/)?', '', staticpath)
staticpath = re.sub('^/', '', staticpath)
resource_is_local = (config.mode != "TRANSLATION" and
(request.host == config.basehost or
request.host == 'www.' + config.basehost)) or \
isIPAddress(request.host) or \
isIPv6Address(request.host) or \
(config.overriderobotstxt and request.uri == '/robots.txt') or \
request.uri.startswith('/antanistaticmap/')
if content_length is not None:
self.bodyProducer.length = int(content_length)
producer = self.bodyProducer
request.headers.removeHeader(b'content-length')
elif transfer_encoding is not None:
producer = self.bodyProducer
request.headers.removeHeader(b'transfer-encoding')
else:
producer = None
if config.mirror is not None:
if config.basehost in config.mirror:
config.mirror.remove(config.basehost)
if len(config.mirror) > 1:
self.var['mirror'] = choice(config.mirror)
elif len(config.mirror) == 1:
self.var['mirror'] = config.mirror[0]
# we serve contents only over https
if not self.isSecure() and (config.transport != 'HTTP'):
self.redirect("https://" + request.host + request.uri)
self.finish()
defer.returnValue(None)
# 0: Request admission control stage
# we try to deny some ua/crawlers regardless the request is (valid or not) / (local or not)
# we deny EVERY request to known user agents reconized with pattern matching
if config.blockcrawl and request.headers.getRawHeaders(b'user-agent') is not None:
for ua in blocked_ua_list:
if re.match(ua, request.headers.getRawHeaders(b'user-agent')[0].lower()):
self.sendError(403, "error_blocked_ua.tpl")
defer.returnValue(NOT_DONE_YET)
# 1: Client capability assessment stage
if request.headers.getRawHeaders(b'accept-encoding') is not None:
if re.search('gzip', request.headers.getRawHeaders(b'accept-encoding')[0]):
self.obj.client_supports_gzip = True
# 2: Content delivery stage
# we need to verify if the requested resource is local (/antanistaticmap/*) or remote
# because some checks must be done only for remote requests;
# in fact local content is always served (css, js, and png in fact are used in errors)
if resource_is_local:
# the requested resource is local, we deliver it directly
try:
if staticpath == "dev/null":
content = "A" * random.randint(20, 1024)
self.setHeader(b'content-type', 'text/plain')
defer.returnValue(self.contentFinish(content))
elif staticpath == "stats/yesterday":
self.setHeader(b'content-type', 'application/json')
content = yield rpc("get_yesterday_stats")
defer.returnValue(self.contentFinish(content))
elif staticpath == "notification":
#################################################################
# Here we need to parse POST data in x-www-form-urlencoded format
#################################################################
content_receiver = BodyReceiver(defer.Deferred())
self.bodyProducer.startProducing(content_receiver)
yield self.bodyProducer.finished
content = ''.join(content_receiver._data)
args = {}
ctype = self.requestHeaders.getRawHeaders(b'content-type')
if ctype is not None:
ctype = ctype[0]
if self.method == b"POST" and ctype:
key, pdict = parse_header(ctype)
if key == b'application/x-www-form-urlencoded':
args.update(parse_qs(content, 1))
#################################################################
if 'by' in args and 'url' in args and 'comment' in args:
tmp = []
tmp.append("From: Tor2web Node %s.%s <%s>\n" % (config.nodename, config.basehost, config.smtpmail))
tmp.append("To: %s\n" % config.smtpmailto_notifications)
tmp.append("Subject: Tor2web Node (IPv4 %s, IPv6 %s): notification for %s\n" % (config.listen_ipv4, config.listen_ipv6, args['url'][0]))
tmp.append("Content-Type: text/plain; charset=ISO-8859-1\n")
tmp.append("Content-Transfer-Encoding: 8bit\n\n")
tmp.append("BY: %s\n" % (args['by'][0]))
tmp.append("URL: %s\n" % (args['url'][0]))
tmp.append("COMMENT: %s\n" % (args['comment'][0]))
message = StringIO(''.join(tmp))
try:
sendmail(config.smtpuser,
config.smtppass,
config.smtpmail,
config.smtpmailto_notifications,
message,
config.smtpdomain,
config.smtpport)
except Exception:
pass
self.setHeader(b'content-type', 'text/plain')
defer.returnValue(self.contentFinish(''))
else:
if type(antanistaticmap[staticpath]) == str:
filename, ext = os.path.splitext(staticpath)
self.setHeader(b'content-type', mimetypes.types_map[ext])
content = antanistaticmap[staticpath]
defer.returnValue(self.contentFinish(content))
elif type(antanistaticmap[staticpath]) == PageTemplate:
defer.returnValue(flattenString(self, antanistaticmap[staticpath]).addCallback(self.contentFinish))
except Exception:
pass
self.sendError(404)
defer.returnValue(NOT_DONE_YET)
else:
self.obj.uri = request.uri
if not request.host:
self.sendError(406, 'error_invalid_hostname.tpl')
defer.returnValue(NOT_DONE_YET)
if config.mode == "TRANSLATION":
self.obj.onion = config.onion
else:
self.obj.onion = request.host.split(".")[0] + ".onion"
rpc_log("detected <onion_url>.tor2web Hostname: %s" % self.obj.onion)
if not verify_onion(self.obj.onion):
self.sendError(406, 'error_invalid_hostname.tpl')
defer.returnValue(NOT_DONE_YET)
if config.mode == "ACCESSLIST":
if not hashlib.md5(self.obj.onion) in access_list:
self.sendError(403, 'error_hs_completely_blocked.tpl')
defer.returnValue(NOT_DONE_YET)
elif config.mode == "BLACKLIST":
if hashlib.md5(self.obj.onion).hexdigest() in access_list:
self.sendError(403, 'error_hs_completely_blocked.tpl')
defer.returnValue(NOT_DONE_YET)
if hashlib.md5(self.obj.onion + self.obj.uri).hexdigest() in access_list:
self.sendError(403, 'error_hs_specific_page_blocked.tpl')
defer.returnValue(NOT_DONE_YET)
# we need to verify if the user is using tor;
# on this condition it's better to redirect on the .onion
if self.getClientIP() in tor_exits_list:
self.redirect("http://" + self.obj.onion + request.uri)
try:
self.finish()
except Exception:
pass
defer.returnValue(None)
# Avoid image hotlinking
if request.uri.lower().endswith(('gif','jpg','png')):
if request.headers.getRawHeaders(b'referer') is not None and \
not config.basehost in request.headers.getRawHeaders(b'referer')[0].lower():
self.sendError(403)
defer.returnValue(NOT_DONE_YET)
# the requested resource is remote, we act as proxy
self.process_request(request)
parsed = urlparse(self.obj.address)
self.var['address'] = self.obj.address
self.var['onion'] = self.obj.onion.replace(".onion", "")
self.var['path'] = parsed[2]
if parsed[3] is not None and parsed[3] != '':
self.var['path'] += '?' + parsed[3]
agent = Agent(reactor, sockhost=config.sockshost, sockport=config.socksport, pool=self.pool)
if config.dummyproxy is None:
proxy_url = 's' + self.obj.address
else:
proxy_url = config.dummyproxy + parsed[2] + '?' + parsed[3]
self.proxy_d = agent.request(self.method,
proxy_url,
self.obj.headers, bodyProducer=producer)
self.proxy_d.addCallback(self.cbResponse)
self.proxy_d.addErrback(self.handleError)
defer.returnValue(NOT_DONE_YET)
def cbResponse(self, response):
self.proxy_response = response
if 600 <= int(response.code) <= 699:
self.setResponseCode(500)
self.var['errorcode'] = int(response.code) - 600
if self.var['errorcode'] in SOCKS_errors:
return flattenString(self, templates[SOCKS_errors[self.var['errorcode']]]).addCallback(self.contentFinish)
else:
return flattenString(self, templates[SOCKS_errors[0x00]]).addCallback(self.contentFinish)
self.setResponseCode(response.code)
self.processResponseHeaders(response.headers)
if response.length is not 0:
finished = defer.Deferred()
if self.obj.html:
response.deliverBody(BodyStreamer(self.handleFixPart, finished))
finished.addCallback(self.handleFixEnd)
else:
response.deliverBody(BodyStreamer(self.handleForwardPart, finished))
finished.addCallback(self.handleForwardEnd)
return finished
else:
self.contentFinish('')
return defer.succeed
def handleHeader(self, key, values):
keyLower = key.lower()
# some headers does not allow multiple occurrences
# in case of multiple occurrences we evaluate only the first
valueLower = values[0].lower()
if keyLower == 'transfer-encoding' and valueLower == 'chunked':
return
elif keyLower == 'content-encoding' and valueLower == 'gzip':
self.obj.server_response_is_gzip = True
return
elif keyLower == 'content-type' and re.search('text/html', valueLower):
self.obj.html = True
elif keyLower == 'content-length':
self.receivedContentLen = valueLower
return
elif keyLower == 'cache-control':
return
if keyLower in 'location':
fixed_values = []
for value in values:
value = re_sub(rexp['t2w'], r'https://\2.' + config.basehost, value)
fixed_values.append(value)
values = fixed_values
self.responseHeaders.setRawHeaders(key, values)
def processResponseHeaders(self, headers):
# currently we track only responding hidden services
# we don't need to block on the rpc now so no yield is needed
rpc("update_stats", str(self.obj.onion.replace(".onion", "")))
for key, values in headers.getAllRawHeaders():
self.handleHeader(key, values)
def connectionLost(self, reason):
try:
if self.proxy_d:
self.proxy_d.cancel()
except Exception:
pass
try:
if self.proxy_response:
self.proxy_response._transport.stopProducing()
except Exception:
pass
try:
http.Request.connectionLost(self, reason)
except Exception:
pass
def finish(self):
try:
http.Request.finish(self)
except Exception:
pass
class FakeRequest(object):
"""
I pretend to be an HTTP request, with a handful of required methods
used by Twisted resource classes (e.g., write, finish,
setResponseCode).
"""
_fluidDB_reqid = None
body = ''
def __init__(self, method, d=None, headers=None, hostname=None):
"""
Initialize. d (if not None) is a deferred that will fire with the
body of the response when request.finish is called.
@param d: A C{Deferred} instance to fire when the request is finished.
@param headers: a C{dict} of headers and values for the request.
"""
self.method = method
self.d = d
self.uri = None
self.requestHeaders = Headers(headers)
self.responseHeaders = Headers()
self.args = {'verb': [method]}
self.hostname = hostname
self.content = StringIO.StringIO()
def isSecure(self):
"""
Mark the request as not being secure (i.e., not HTTPS). The
isSecure is used by the FluidDB wsfe to construct the Location
header in the response to a POST.
@return: A constant, C{False}, seeing as we don't care about
the return value, only that we provide the method.
"""
return False
def finish(self):
"""
Indicate that the request has been fully serviced. Send the
response body (as accumulated by self.write) back to the deferred
(if any) we were passed in __init__.
@return: C{None}.
"""
if self.d:
self.d.callback(self.body)
def write(self, data):
"""
Fake the writing of data back to the client. Instead, we
accumuluate it so as to deliver it to our test client when
self.finish is called.
@param data: A C{str} of response payload data.
@return: C{None}.
"""
self.body += data
def getHeader(self, key):
"""
Get a header from the request. This is copied from Twisted's
twisted.web.http.Request class.
@param key: A C{str} indicating the header to return.
@return: The C{str} header value from the request if it exists,
else C{None}.
"""
value = self.requestHeaders.getRawHeaders(key)
if value is not None:
return value[-1]
def getResponseHeader(self, key):
"""
Get a header from the response.
@param key: A C{str} indicating the header to return.
@return: The C{str} header value from the response if it exists,
else C{None}.
"""
value = self.responseHeaders.getRawHeaders(key)
if value is not None:
return value[-1]
def setHeader(self, name, value):
"""
Set a header for the HTTP response. This is copied from Twisted's
twisted.web.http.Request class.
@param name: A C{str} indicating the header to set.
@param value: A C{str} values for the header.
@return: C{None}.
"""
self.responseHeaders.setRawHeaders(name, [value])
def getAllHeaders(self):
"""
Get all the request headers. This is copied from Twisted's
twisted.web.http.Request class.
@return: A C{dict} of request header name -> value.
"""
headers = {}
for k, v in self.requestHeaders.getAllRawHeaders():
headers[k.lower()] = v[-1]
return headers
def setResponseCode(self, code):
"""
Set the response status code.
@param code: An HTTP status code.
@return: C{None}.
"""
self.status = code
def notifyFinish(self):
"""
Return a C{twisted.internet.Deferred} that fires when the request
finishes or errors if the client disconnects. Note that this method
is needed as resource.py calls it, but we do not need to actually
fire the returned deferred for our tests (which cause synchronous
exceptions to immediately be returned to the request errback).
@return: C{twisted.internet.Deferred}
"""
return defer.succeed(None)
def getRequestHostname(self):
"""
Return the hostname that was used for issuing this request.
"""
return self.hostname
class FakeRequest(object):
"""
A fake request suitable for use in place of C{twisted.web.http.Request}.
@param method: The HTTP method being invoked such as C{GET} or C{POST}.
@param uri: The URI of the resource being requested.
@param headers: A C{dict} of headers to send in the request.
@param args: A C{dict} of arguments that should be appended to the URI in
the request.
"""
content = None
def __init__(self, method, uri, headers=None, args=None):
self.method = method
self.uri = uri
self.requestHeaders = Headers(headers)
self.responseHeaders = Headers()
self.args = {'verb': [method]}
if args:
self.args.update(args)
self._fluidDB_reqid = 'xxx'
self.finished = False
def finish(self):
"""
Indicate that all response data has been written to this request.
"""
self.finished = True
def getHeader(self, key):
"""
Get the value of an HTTP request header.
@param key: The name of the HTTP header to retrieve a value for.
@return: The value set for the header or C{None} if a value is not
available.
"""
value = self.requestHeaders.getRawHeaders(key)
if value is not None:
return value[-1]
def getResponseHeader(self, key):
"""
Get the value of an HTTP response header.
@param key: The name of the HTTP header to retrieve a value for.
@return: The value set for the header or C{None} if a value is not
available.
"""
value = self.responseHeaders.getRawHeaders(key)
if value is not None:
return value[-1]
def setHeader(self, name, value):
"""
Set an HTTP header to include in the response returned to the client.
@param name: The name of the header to set.
@param value: The value to set.
"""
self.responseHeaders.setRawHeaders(name, [value])
def getAllHeaders(self):
"""
Return all the request headers.
@return: A C{dict} of all headers, with their values.
"""
headers = {}
for k, v in self.requestHeaders.getAllRawHeaders():
headers[k.lower()] = v[-1]
return headers
def setResponseCode(self, code):
"""
Set the HTTP response code to return to the client.
@param code: An HTTP response code as defined in C{twisted.web.http}.
"""
self.status = code
def notifyFinish(self):
"""
Return a C{twisted.internet.Deferred} that fires when the request
finishes or errors if the client disconnects. Note that this method
is needed as resource.py calls it, but we do not need to actually
fire the returned deferred for our test (which causes a synchronous
exception to immediately be returned to the request errback).
@return: C{twisted.internet.Deferred}
"""
return defer.succeed(None)
class FakeRequest(Componentized):
"""
Implementation of L{inevow.IRequest} which is convenient to use in unit
tests.
@ivar lastModified: The value passed to L{setLastModified} or C{None} if
that method has not been called.
@type accumulator: C{str}
@ivar accumulator: The bytes written to the response body.
@type deferred: L{Deferred}
@ivar deferred: The deferred which represents rendering of the response
to this request. This is basically an implementation detail of
L{NevowRequest}. Application code should probably never use this.
@ivar _appRootURL: C{None} or the object passed to L{rememberRootURL}.
"""
fields = None
failure = None
context = None
redirected_to = None
lastModified = None
content = ""
method = 'GET'
code = http.OK
deferred = None
accumulator = ''
_appRootURL = None
def __init__(self,
headers=None,
args=None,
avatar=None,
uri='/',
currentSegments=None,
cookies=None,
user="",
password="",
isSecure=False):
"""
Create a FakeRequest instance.
@param headers: dict of request headers
@param args: dict of args
@param avatar: avatar to pass to the FakeSession instance
@param uri: request URI
@param currentSegments: list of segments that have "already been located"
@param cookies: dict of cookies
@param user: username (like in http auth)
@param password: password (like in http auth)
@param isSecure: whether this request represents an HTTPS url
"""
Componentized.__init__(self)
self.uri = uri
if not uri.startswith('/'):
raise ValueError('uri must be relative with absolute path')
self.path = uri
self.prepath = []
postpath = uri.split('?')[0]
assert postpath.startswith('/')
self.postpath = postpath[1:].split('/')
if currentSegments is not None:
for seg in currentSegments:
assert seg == self.postpath[0]
self.prepath.append(self.postpath.pop(0))
else:
self.prepath.append('')
self.responseHeaders = Headers()
self.args = args or {}
self.sess = FakeSession(avatar)
self.site = FakeSite()
self.requestHeaders = Headers()
if headers:
for k, v in headers.items():
self.requestHeaders.setRawHeaders(k, [v])
if cookies is not None:
self.cookies = cookies
else:
self.cookies = {}
self.user = user
self.password = password
self.secure = isSecure
self.deferred = defer.Deferred()
def URLPath(self):
from nevow import url
return url.URL.fromContext(self)
def getSession(self):
return self.sess
def registerProducer(self, producer, streaming):
"""
Synchronously cause the given producer to produce all of its data.
This will not work with push producers. Do not use it with them.
"""
keepGoing = [None]
self.unregisterProducer = keepGoing.pop
while keepGoing:
producer.resumeProducing()
del self.unregisterProducer
def v():
def get(self):
return self.accumulator
return get,
v = property(*v())
def write(self, bytes):
"""
Accumulate the given bytes as part of the response body.
@type bytes: C{str}
"""
self.accumulator += bytes
finished = False
def finishRequest(self, success):
self.finished = True
def finish(self):
self.deferred.callback('')
def getHeader(self, key):
return self.requestHeaders.getRawHeaders(key, [None])[0]
def setHeader(self, key, val):
self.responseHeaders.setRawHeaders(key, [val])
def redirect(self, url):
self.redirected_to = url
def processingFailed(self, f):
self.failure = f
def setResponseCode(self, code):
self.code = code
def setLastModified(self, when):
self.lastModified = when
def prePathURL(self):
"""
The absolute URL up until the last handled segment of this request.
@rtype: C{str}.
"""
return 'http://%s/%s' % (self.getHeader('host')
or 'localhost', '/'.join(self.prepath))
def getClientIP(self):
return '127.0.0.1'
def addCookie(self,
k,
v,
expires=None,
domain=None,
path=None,
max_age=None,
comment=None,
secure=None):
"""
Set a cookie for use in subsequent requests.
"""
self.cookies[k] = v
def getCookie(self, k):
"""
Fetch a cookie previously set.
"""
return self.cookies.get(k)
def getUser(self):
"""
Returns the HTTP auth username.
"""
return self.user
def getPassword(self):
"""
Returns the HTTP auth password.
"""
return self.password
def getRootURL(self):
"""
Return the previously remembered URL.
"""
return self._appRootURL
def rememberRootURL(self, url=None):
"""
For compatibility with appserver.NevowRequest.
"""
if url is None:
raise NotImplementedError(
"Default URL remembering logic is not implemented.")
self._appRootURL = url
def isSecure(self):
"""
Returns whether this is an HTTPS request or not.
"""
return self.secure
def _warnHeaders(self, old, new):
"""
Emit a warning related to use of one of the deprecated C{headers} or
C{received_headers} attributes.
@param old: The name of the deprecated attribute to which the warning
pertains.
@param new: The name of the preferred attribute which replaces the old
attribute.
"""
warnings.warn(
category=DeprecationWarning,
message=("nevow.testutil.FakeRequest.%(old)s was deprecated in "
"Nevow 0.13.0: Please use nevow.testutil.FakeRequest."
"%(new)s instead." % dict(old=old, new=new)),
stacklevel=3)
@property
def headers(self):
"""
Transform the L{Headers}-style C{responseHeaders} attribute into a
deprecated C{dict}-style C{headers} attribute.
"""
self._warnHeaders("headers", "responseHeaders")
return appserver._DictHeaders(self.responseHeaders)
@property
def received_headers(self):
"""
Transform the L{Headers}-style C{requestHeaders} attribute into a
deprecated C{dict}-style C{received_headers} attribute.
"""
self._warnHeaders("received_headers", "requestHeaders")
return appserver._DictHeaders(self.requestHeaders)
class DummyRequest(object):
"""
Represents a dummy or fake request. See L{twisted.web.server.Request}.
@ivar _finishedDeferreds: L{None} or a C{list} of L{Deferreds} which will
be called back with L{None} when C{finish} is called or which will be
errbacked if C{processingFailed} is called.
@type requestheaders: C{Headers}
@ivar requestheaders: A Headers instance that stores values for all request
headers.
@type responseHeaders: C{Headers}
@ivar responseHeaders: A Headers instance that stores values for all
response headers.
@type responseCode: C{int}
@ivar responseCode: The response code which was passed to
C{setResponseCode}.
@type written: C{list} of C{bytes}
@ivar written: The bytes which have been written to the request.
"""
uri = b'http://dummy/'
method = b'GET'
client = None
def registerProducer(self, prod, s):
self.go = 1
while self.go:
prod.resumeProducing()
def unregisterProducer(self):
self.go = 0
def __init__(self, postpath, session=None):
self.sitepath = []
self.written = []
self.finished = 0
self.postpath = postpath
self.prepath = []
self.session = None
self.protoSession = session or Session(0, self)
self.args = {}
self.requestHeaders = Headers()
self.responseHeaders = Headers()
self.responseCode = None
self._finishedDeferreds = []
self._serverName = b"dummy"
self.clientproto = b"HTTP/1.0"
def getAllHeaders(self):
"""
Return dictionary mapping the names of all received headers to the last
value received for each.
Since this method does not return all header information,
C{self.requestHeaders.getAllRawHeaders()} may be preferred.
NOTE: This function is a direct copy of
C{twisted.web.http.Request.getAllRawHeaders}.
"""
headers = {}
for k, v in self.requestHeaders.getAllRawHeaders():
headers[k.lower()] = v[-1]
return headers
def getHeader(self, name):
"""
Retrieve the value of a request header.
@type name: C{bytes}
@param name: The name of the request header for which to retrieve the
value. Header names are compared case-insensitively.
@rtype: C{bytes} or L{None}
@return: The value of the specified request header.
"""
return self.requestHeaders.getRawHeaders(name.lower(), [None])[0]
def setHeader(self, name, value):
"""TODO: make this assert on write() if the header is content-length
"""
self.responseHeaders.addRawHeader(name, value)
def getSession(self):
if self.session:
return self.session
assert not self.written, "Session cannot be requested after data has been written."
self.session = self.protoSession
return self.session
def render(self, resource):
"""
Render the given resource as a response to this request.
This implementation only handles a few of the most common behaviors of
resources. It can handle a render method that returns a string or
C{NOT_DONE_YET}. It doesn't know anything about the semantics of
request methods (eg HEAD) nor how to set any particular headers.
Basically, it's largely broken, but sufficient for some tests at least.
It should B{not} be expanded to do all the same stuff L{Request} does.
Instead, L{DummyRequest} should be phased out and L{Request} (or some
other real code factored in a different way) used.
"""
result = resource.render(self)
if result is NOT_DONE_YET:
return
self.write(result)
self.finish()
def write(self, data):
if not isinstance(data, bytes):
raise TypeError("write() only accepts bytes")
self.written.append(data)
def notifyFinish(self):
"""
Return a L{Deferred} which is called back with L{None} when the request
is finished. This will probably only work if you haven't called
C{finish} yet.
"""
finished = Deferred()
self._finishedDeferreds.append(finished)
return finished
def finish(self):
"""
Record that the request is finished and callback and L{Deferred}s
waiting for notification of this.
"""
self.finished = self.finished + 1
if self._finishedDeferreds is not None:
observers = self._finishedDeferreds
self._finishedDeferreds = None
for obs in observers:
obs.callback(None)
def processingFailed(self, reason):
"""
Errback and L{Deferreds} waiting for finish notification.
"""
if self._finishedDeferreds is not None:
observers = self._finishedDeferreds
self._finishedDeferreds = None
for obs in observers:
obs.errback(reason)
def addArg(self, name, value):
self.args[name] = [value]
def setResponseCode(self, code, message=None):
"""
Set the HTTP status response code, but takes care that this is called
before any data is written.
"""
assert not self.written, "Response code cannot be set after data has been written: %s." % "@@@@".join(
self.written)
self.responseCode = code
self.responseMessage = message
def setLastModified(self, when):
assert not self.written, "Last-Modified cannot be set after data has been written: %s." % "@@@@".join(
self.written)
def setETag(self, tag):
assert not self.written, "ETag cannot be set after data has been written: %s." % "@@@@".join(
self.written)
def getClientIP(self):
"""
Return the IPv4 address of the client which made this request, if there
is one, otherwise L{None}.
"""
if isinstance(self.client, IPv4Address):
return self.client.host
return None
def getRequestHostname(self):
"""
Get a dummy hostname associated to the HTTP request.
@rtype: C{bytes}
@returns: a dummy hostname
"""
return self._serverName
def getHost(self):
"""
Get a dummy transport's host.
@rtype: C{IPv4Address}
@returns: a dummy transport's host
"""
return IPv4Address('TCP', '127.0.0.1', 80)
def setHost(self, host, port, ssl=0):
"""
Change the host and port the request thinks it's using.
@type host: C{bytes}
@param host: The value to which to change the host header.
@type ssl: C{bool}
@param ssl: A flag which, if C{True}, indicates that the request is
considered secure (if C{True}, L{isSecure} will return C{True}).
"""
self._forceSSL = ssl # set first so isSecure will work
if self.isSecure():
default = 443
else:
default = 80
if port == default:
hostHeader = host
else:
hostHeader = host + b":" + intToBytes(port)
self.requestHeaders.addRawHeader(b"host", hostHeader)
def getClient(self):
"""
Get the client's IP address, if it has one.
@return: The same value as C{getClientIP}.
@rtype: L{bytes}
"""
return self.getClientIP()
def redirect(self, url):
"""
Utility function that does a redirect.
The request should have finish() called after this.
"""
self.setResponseCode(FOUND)
self.setHeader(b"location", url)
class FakeRequest(object):
"""
A fake C{twisted.web.http.Request} implementation, suitable for use in
tests.
"""
def __init__(self, args=None, postpath=None, prepath=None, path=None,
uri=None, method='GET', headers=None, body=''):
self.args = {} if args is None else args
self.written = StringIO()
self.finished = False
self.code = None
self.method = method
self.path = path
self.postpath = postpath
self.prepath = prepath
self.requestHeaders = Headers({}) if headers is None else headers
self.responseHeaders = Headers({})
self.uri = uri
self._fluidDB_reqid = requestId()
self.content = StringIO(body)
def write(self, content):
"""Write data as a result of an HTTP request.
@param content: A C{str} containing the bytes to send as part of the
response body.
"""
if not isinstance(content, str):
raise RuntimeError('Only strings can be written.')
self.written.write(content)
def finish(self):
"""Indicate that all response data has been written to this request."""
if self.code is None:
self.code = 200
self.finished = True
def setResponseCode(self, code):
"""Set the HTTP response code.
@param code: An C{int} HTTP status code.
"""
self.code = code
def setHeader(self, key, value):
"""Set an HTTP response header.
@param key: The name of the response header.
@param value: The value for the response header.
"""
self.responseHeaders.setRawHeaders(key, [value])
def getHeader(self, key):
"""
Get a header from the request. This is copied from Twisted's
C{twisted.web.http.Request} class.
@param key: A C{str} indicating the header to return.
@return: The C{str} header value from the request if it exists,
else C{None}.
"""
value = self.requestHeaders.getRawHeaders(key)
if value is not None:
return value[-1]
def getResponseHeader(self, key):
"""Get a header from the response.
@param key: A C{str} indicating the header to return.
@return: The C{str} header value from the response if it exists,
else C{None}.
"""
value = self.responseHeaders.getRawHeaders(key)
if value is not None:
return value[-1]
@property
def response(self):
"""The HTTP response body."""
return self.written.getvalue()
def getAllHeaders(self):
"""
Get all the request headers. This is copied from Twisted's
C{twisted.web.http.Request} class.
@return: A C{dict} of request header name -> value.
"""
headers = {}
for k, v in self.requestHeaders.getAllRawHeaders():
headers[k.lower()] = v[-1]
return headers
def notifyFinish(self):
"""
Return a C{twisted.internet.Deferred} that fires when the request
finishes or errors if the client disconnects. Note that this method
is needed as resource.py calls it, but we do not need to actually
fire the returned deferred for our tests (which cause synchronous
exceptions to immediately be returned to the request errback).
@return: A C{Deferred} as just described.
"""
return succeed(None)
def isSecure(self):
"""Is the request secure?
@return: a C{bool} that is C{True} if the request is secure.
"""
# The way we tell if a request is secure is based on a header set
# for us by nginx. Our Twisted web service only handles plain http
# requests that are forwarded to it by nginx (via haproxy).
secureHeader = self.getHeader('X-Forwarded-Protocol')
return (secureHeader == 'https')
