def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a new request via the configured proxy.
"""
if version >= Version('twisted', 13, 1, 0):
parsed_uri = _URI.getFromBytes(uri)
scheme = parsed_uri.scheme
host = parsed_uri.host
port = parsed_uri.port
else:
scheme, host, port, path = _parse(uri)
request_path = uri
d = self._connect(scheme, host, port)
if headers is None:
headers = Headers()
if not headers.hasHeader('host'):
# This is a lot of copying. It might be nice if there were a bit
# less.
headers = Headers(dict(headers.getAllRawHeaders()))
headers.addRawHeader(
'host', self._computeHostValue(scheme, host, port))
def cbConnected(proto):
# NOTE: For the proxy case the path should be the full URI.
return proto.request(
Request(method, request_path, headers, bodyProducer))
d.addCallback(cbConnected)
return d
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a new request via the configured proxy.
"""
scheme, host, port, path = _parse(uri)
request_path = uri
d = self._connect(scheme, host, port)
if headers is None:
headers = Headers()
if not headers.hasHeader('host'):
# This is a lot of copying. It might be nice if there were a bit
# less.
headers = Headers(dict(headers.getAllRawHeaders()))
headers.addRawHeader('host',
self._computeHostValue(scheme, host, port))
def cbConnected(proto):
# NOTE: For the proxy case the path should be the full URI.
return proto.request(
Request(method, request_path, headers, bodyProducer))
d.addCallback(cbConnected)
return d
def test_removeHeaderDoesntExist(self):
"""
L{Headers.removeHeader} is a no-operation when the specified header is
not found.
"""
h = Headers()
h.removeHeader(b"test")
self.assertEqual(list(h.getAllRawHeaders()), [])
def test_removeHeaderDoesntExist(self):
"""
L{Headers.removeHeader} is a no-operation when the specified header is
not found.
"""
h = Headers()
h.removeHeader("test")
self.assertEqual(list(h.getAllRawHeaders()), [])
def request(self, method, path, headers=None, body=None, expected_code=200,
retry=0, delay=0):
"""
Send an http request to the gerrit service for the given path.
If 'retry' is specified, transient errors (http response code 500-599) will
be retried after an exponentially-increasing delay.
Returns a Deferred which will call back with the parsed json body of the
gerrit server's response.
Args:
method: 'GET', 'POST', etc.
path: Path element of the url.
headers: dict of http request headers.
body: json-encodable body of http request.
expected_code: http response code expected in reply.
retry: How many times to retry transient errors.
delay: Wait this many seconds before sending the request.
"""
retry_delay = delay * 2 if delay else 0.5
retry_args = (
method, path, headers, body, expected_code, retry - 1, retry_delay)
if not path.startswith('/'):
path = '/' + path
if not headers:
headers = Headers()
else:
# Make a copy so mutations don't affect retry attempts.
headers = Headers(dict(headers.getAllRawHeaders()))
if self.auth_token:
if not path.startswith('/a/'):
path = '/a' + path
headers.setRawHeaders('Authorization', [self.auth_token])
url = '%s://%s%s' % (self.gerrit_protocol, self.gerrit_host, path)
if body:
body = JsonBodyProducer(json.dumps(body))
headers.setRawHeaders('Content-Type', ['application/json'])
if DEBUG:
log.msg(url)
if delay:
d = defer.succeed(None)
d.addCallback(
reactor.callLater, delay, Agent.request, self, method, str(url),
headers, body)
else:
d = Agent.request(self, method, str(url), headers, body)
def _check_code(response):
if response.code == expected_code:
return response
if retry > 0 and response.code >= 500 and response.code < 600:
return self.request(*retry_args)
msg = 'Failed gerrit request (code %s, expected %s): %s' % (
response.code, expected_code, url)
raise GerritError(msg, response.code)
d.addCallback(_check_code)
d.addCallback(JsonResponse.Get, url=url)
return d
def _respondToChallenge(self, challenge, method, uri, headers, bodyProducer):
if headers is None:
headers = Headers()
else:
headers = Headers(dict(headers.getAllRawHeaders()))
for k, vs in challenge.response(uri, method, self._authinfo).iteritems():
for v in vs:
headers.addRawHeader(k, v)
return self._agent.request(method, uri, headers, bodyProducer)
def _respondToChallenge(self, challenge, method, uri, headers, bodyProducer):
if headers is None:
headers = Headers()
else:
headers = Headers(dict(headers.getAllRawHeaders()))
for k, vs in challenge.response(uri, method, self._authinfo).iteritems():
for v in vs:
headers.addRawHeader(k, v)
return self._agent.request(method, uri, headers, bodyProducer)
def test_getAllRawHeaders(self):
"""
L{Headers.getAllRawHeaders} returns an iterable of (k, v) pairs, where
C{k} is the canonicalized representation of the header name, and C{v}
is a sequence of values.
"""
h = Headers()
h.setRawHeaders(b"test", [b"lemurs"])
h.setRawHeaders(b"www-authenticate", [b"basic aksljdlk="])
allHeaders = set([(k, tuple(v)) for k, v in h.getAllRawHeaders()])
self.assertEqual(allHeaders, set([(b"WWW-Authenticate", (b"basic aksljdlk=",)), (b"Test", (b"lemurs",))]))
def test_header_key_not_present(self):
"""
When the header key isn't present, match() should return a mismatch
with the correct description and some details.
"""
matcher = HasHeader('test', ['abc'])
headers = Headers({'Something else': ['abc']})
match = matcher.match(headers)
assert_that(match.describe(),
Equals('The response does not have a "test" header'))
headers_content = text_content(repr(dict(headers.getAllRawHeaders())))
assert_that(match.get_details(),
Equals({'raw headers': headers_content}))
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a new request.
@param method: The request method to send.
@type method: C{str}
@param uri: The request URI send.
@type uri: C{str}
@param scheme: A string like C{'http'} or C{'https'} (the only two
supported values) to use to determine how to establish the
connection.
@param host: A C{str} giving the hostname which will be connected to in
order to issue a request.
@param port: An C{int} giving the port number the connection will be on.
@param path: A C{str} giving the path portion of the request URL.
@param headers: The request headers to send. If no I{Host} header is
included, one will be added based on the request URI.
@type headers: L{Headers}
@param bodyProducer: An object which will produce the request body or,
if the request body is to be empty, L{None}.
@type bodyProducer: L{IBodyProducer} provider
@return: A L{Deferred} which fires with the result of the request (a
L{Response} instance), or fails if there is a problem setting up a
connection over which to issue the request. It may also fail with
L{SchemeNotSupported} if the scheme of the given URI is not
supported.
@rtype: L{Deferred}
"""
scheme, host, port, path = _parse(uri)
if headers is None:
headers = Headers()
if not headers.hasHeader('host'):
# This is a lot of copying. It might be nice if there were a bit
# less.
headers = Headers(dict(headers.getAllRawHeaders()))
headers.addRawHeader(
'host', self._computeHostValue(scheme, host, port))
if self.persistent:
sem = self._semaphores.get((scheme, host, port))
if sem is None:
sem = DeferredSemaphore(self.maxConnectionsPerHostName)
self._semaphores[scheme, host, port] = sem
return sem.run(self._request, method, scheme, host, port, path,
headers, bodyProducer)
else:
return self._request(
method, scheme, host, port, path, headers, bodyProducer)
def test_getAllRawHeaders(self):
"""
L{Headers.getAllRawHeaders} returns an iterable of (k, v) pairs, where
C{k} is the canonicalized representation of the header name, and C{v}
is a sequence of values.
"""
h = Headers()
h.setRawHeaders("test", ["lemurs"])
h.setRawHeaders("www-authenticate", ["basic aksljdlk="])
allHeaders = set([(k, tuple(v)) for k, v in h.getAllRawHeaders()])
self.assertEqual(allHeaders,
set([("WWW-Authenticate", ("basic aksljdlk=",)),
("Test", ("lemurs",))]))
def test_getAllRawHeaders(self):
"""
L{Headers.getAllRawHeaders} returns an iterable of (k, v) pairs, where
C{k} is the canonicalized representation of the header name, and C{v}
is a sequence of values.
"""
h = Headers()
h.setRawHeaders(u"test\u00E1", [u"lemurs"])
h.setRawHeaders(u"www-authenticate", [u"basic aksljdlk="])
h.setRawHeaders(u"content-md5", [u"kjdfdfgdfgnsd"])
allHeaders = set([(k, tuple(v)) for k, v in h.getAllRawHeaders()])
self.assertEqual(allHeaders,
set([(b"WWW-Authenticate", (b"basic aksljdlk=",)),
(b"Content-MD5", (b"kjdfdfgdfgnsd",)),
(b"Test\xe1", (b"lemurs",))]))
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a new request.
@param method: The request method to send.
@type method: C{str}
@param uri: The request URI send.
@type uri: C{str}
@param headers: The request headers to send. If no I{Host} header is
included, one will be added based on the request URI.
@type headers: L{Headers}
@param bodyProducer: An object which will produce the request body or,
if the request body is to be empty, L{None}.
@type bodyProducer: L{IBodyProducer} provider
@return: A L{Deferred} which fires with the result of the request (a
L{Response} instance), or fails if there is a problem setting up a
connection over which to issue the request. It may also fail with
L{SchemeNotSupported} if the scheme of the given URI is not
supported.
@rtype: L{Deferred}
"""
scheme, host, port, path = _parse(uri)
if scheme != 'http':
return defer.fail(
SchemeNotSupported("Unsupported scheme: %r" % (scheme, )))
cc = ClientCreator(self._reactor, self._protocol)
d = cc.connectTCP(host, port)
if headers is None:
headers = Headers()
if not headers.hasHeader('host'):
# This is a lot of copying. It might be nice if there were a bit
# less.
headers = Headers(dict(headers.getAllRawHeaders()))
headers.addRawHeader('host',
self._computeHostValue(scheme, host, port))
def cbConnected(proto):
return proto.request(Request(method, path, headers, bodyProducer))
d.addCallback(cbConnected)
return d
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a new request.
@param method: The request method to send.
@type method: C{str}
@param uri: The request URI send.
@type uri: C{str}
@param headers: The request headers to send. If no I{Host} header is
included, one will be added based on the request URI.
@type headers: L{Headers}
@param bodyProducer: An object which will produce the request body or,
if the request body is to be empty, L{None}.
@type bodyProducer: L{IBodyProducer} provider
@return: A L{Deferred} which fires with the result of the request (a
L{Response} instance), or fails if there is a problem setting up a
connection over which to issue the request. It may also fail with
L{SchemeNotSupported} if the scheme of the given URI is not
supported.
@rtype: L{Deferred}
"""
scheme, host, port, path = _parse(uri)
if scheme != 'http':
return defer.fail(SchemeNotSupported(
"Unsupported scheme: %r" % (scheme,)))
cc = ClientCreator(self._reactor, self._protocol)
d = cc.connectTCP(host, port)
if headers is None:
headers = Headers()
if not headers.hasHeader('host'):
# This is a lot of copying. It might be nice if there were a bit
# less.
headers = Headers(dict(headers.getAllRawHeaders()))
headers.addRawHeader(
'host', self._computeHostValue(scheme, host, port))
def cbConnected(proto):
return proto.request(Request(method, path, headers, bodyProducer))
d.addCallback(cbConnected)
return d
def request(self, method, uri, headers=None, bodyProducer=None):
"""
Issue a new request via the configured proxy.
"""
request_path = uri
d = self._connect(None, None, None)
if headers is None:
headers = Headers()
if not headers.hasHeader('host'):
# This is a lot of copying. It might be nice if there were a bit
# less.
headers = Headers(dict(headers.getAllRawHeaders()))
headers.addRawHeader(
'host', self._computeHostValue(scheme, host, port))
def cbConnected(proto):
# NOTE: For the proxy case the path should be the full URI.
return proto.request(Request(method, request_path, headers, bodyProducer))
d.addCallback(cbConnected)
return d
def do_send():
headers_dict = {
'Content-Type': ['application/x-www-form-urlencoded'],
'Host': [self.server[8:]]
}
if not self.oauth_header is None:
headers_dict['Authorization'] = [self.oauth_header]
logging.debug("headers dictionary: %s", headers_dict)
logging.debug("message: %s", str(message))
headers = Headers(headers_dict)
logging.debug("headers object:")
for header in headers.getAllRawHeaders():
logging.debug("> %s", header)
d = self.agent.request('POST', self.server, headers,
BayeuxProducer(str(message)))
logging.debug("send_message.do_send(): d object:\n%s", str(d))
def cb(response):
logging.debug(
"send_message.do_send.cb(): response version: %s",
response.version)
logging.debug("send_message.do_send.cb(): response code: %s",
response.code)
logging.debug("send_message.do_send.cb(): response phrase: %s",
response.phrase)
logging.debug(
"send_message.do_send.cb(): response headers:\n%s",
pprint.pformat(list(response.headers.getAllRawHeaders())))
response.deliverBody(self.receiver)
return d
def error(reason):
logging.error('Error sending msg: %s' % reason)
logging.error(reason.getErrorMessage())
logging.debug(reason.value.reasons[0].printTraceback())
if errback is not None:
errback(reason)
d.addCallback(cb)
d.addErrback(error)
def do_send():
headers_dict = {
'Content-Type': ['application/x-www-form-urlencoded'],
'Host': [self.server[8:]]
}
if not self.oauth_header is None:
headers_dict['Authorization'] = [self.oauth_header]
logging.debug("headers dictionary: %s", headers_dict)
logging.debug("message: %s", str(message))
headers = Headers(headers_dict)
logging.debug("headers object:")
for header in headers.getAllRawHeaders():
logging.debug("> %s", header)
d = self.agent.request('POST',
self.server,
headers,
BayeuxProducer(str(message)))
logging.debug("send_message.do_send(): d object:\n%s", str(d))
def cb(response):
logging.debug("send_message.do_send.cb(): response version: %s", response.version)
logging.debug("send_message.do_send.cb(): response code: %s", response.code)
logging.debug("send_message.do_send.cb(): response phrase: %s", response.phrase)
logging.debug("send_message.do_send.cb(): response headers:\n%s", pprint.pformat(list(response.headers.getAllRawHeaders())))
response.deliverBody(self.receiver)
return d
def error(reason):
logging.error('Error sending msg: %s' % reason)
logging.error(reason.getErrorMessage())
logging.debug(reason.value.reasons[0].printTraceback())
if errback is not None:
errback(reason)
d.addCallback(cb)
d.addErrback(error)
class DummyRequest(object):
"""
Represents a dummy or fake request. See L{twisted.web.server.Request}.
@ivar _finishedDeferreds: L{None} or a C{list} of L{Deferreds} which will
be called back with L{None} when C{finish} is called or which will be
errbacked if C{processingFailed} is called.
@type requestheaders: C{Headers}
@ivar requestheaders: A Headers instance that stores values for all request
headers.
@type responseHeaders: C{Headers}
@ivar responseHeaders: A Headers instance that stores values for all
response headers.
@type responseCode: C{int}
@ivar responseCode: The response code which was passed to
C{setResponseCode}.
@type written: C{list} of C{bytes}
@ivar written: The bytes which have been written to the request.
"""
uri = b'http://dummy/'
method = b'GET'
client = None
def registerProducer(self, prod, s):
self.go = 1
while self.go:
prod.resumeProducing()
def unregisterProducer(self):
self.go = 0
def __init__(self, postpath, session=None):
self.sitepath = []
self.written = []
self.finished = 0
self.postpath = postpath
self.prepath = []
self.session = None
self.protoSession = session or Session(0, self)
self.args = {}
self.requestHeaders = Headers()
self.responseHeaders = Headers()
self.responseCode = None
self._finishedDeferreds = []
self._serverName = b"dummy"
self.clientproto = b"HTTP/1.0"
def getAllHeaders(self):
"""
Return dictionary mapping the names of all received headers to the last
value received for each.
Since this method does not return all header information,
C{self.requestHeaders.getAllRawHeaders()} may be preferred.
NOTE: This function is a direct copy of
C{twisted.web.http.Request.getAllRawHeaders}.
"""
headers = {}
for k, v in self.requestHeaders.getAllRawHeaders():
headers[k.lower()] = v[-1]
return headers
def getHeader(self, name):
"""
Retrieve the value of a request header.
@type name: C{bytes}
@param name: The name of the request header for which to retrieve the
value. Header names are compared case-insensitively.
@rtype: C{bytes} or L{None}
@return: The value of the specified request header.
"""
return self.requestHeaders.getRawHeaders(name.lower(), [None])[0]
def setHeader(self, name, value):
"""TODO: make this assert on write() if the header is content-length
"""
self.responseHeaders.addRawHeader(name, value)
def getSession(self):
if self.session:
return self.session
assert not self.written, "Session cannot be requested after data has been written."
self.session = self.protoSession
return self.session
def render(self, resource):
"""
Render the given resource as a response to this request.
This implementation only handles a few of the most common behaviors of
resources. It can handle a render method that returns a string or
C{NOT_DONE_YET}. It doesn't know anything about the semantics of
request methods (eg HEAD) nor how to set any particular headers.
Basically, it's largely broken, but sufficient for some tests at least.
It should B{not} be expanded to do all the same stuff L{Request} does.
Instead, L{DummyRequest} should be phased out and L{Request} (or some
other real code factored in a different way) used.
"""
result = resource.render(self)
if result is NOT_DONE_YET:
return
self.write(result)
self.finish()
def write(self, data):
if not isinstance(data, bytes):
raise TypeError("write() only accepts bytes")
self.written.append(data)
def notifyFinish(self):
"""
Return a L{Deferred} which is called back with L{None} when the request
is finished. This will probably only work if you haven't called
C{finish} yet.
"""
finished = Deferred()
self._finishedDeferreds.append(finished)
return finished
def finish(self):
"""
Record that the request is finished and callback and L{Deferred}s
waiting for notification of this.
"""
self.finished = self.finished + 1
if self._finishedDeferreds is not None:
observers = self._finishedDeferreds
self._finishedDeferreds = None
for obs in observers:
obs.callback(None)
def processingFailed(self, reason):
"""
Errback and L{Deferreds} waiting for finish notification.
"""
if self._finishedDeferreds is not None:
observers = self._finishedDeferreds
self._finishedDeferreds = None
for obs in observers:
obs.errback(reason)
def addArg(self, name, value):
self.args[name] = [value]
def setResponseCode(self, code, message=None):
"""
Set the HTTP status response code, but takes care that this is called
before any data is written.
"""
assert not self.written, "Response code cannot be set after data has been written: %s." % "@@@@".join(
self.written)
self.responseCode = code
self.responseMessage = message
def setLastModified(self, when):
assert not self.written, "Last-Modified cannot be set after data has been written: %s." % "@@@@".join(
self.written)
def setETag(self, tag):
assert not self.written, "ETag cannot be set after data has been written: %s." % "@@@@".join(
self.written)
def getClientIP(self):
"""
Return the IPv4 address of the client which made this request, if there
is one, otherwise L{None}.
"""
if isinstance(self.client, IPv4Address):
return self.client.host
return None
def getRequestHostname(self):
"""
Get a dummy hostname associated to the HTTP request.
@rtype: C{bytes}
@returns: a dummy hostname
"""
return self._serverName
def getHost(self):
"""
Get a dummy transport's host.
@rtype: C{IPv4Address}
@returns: a dummy transport's host
"""
return IPv4Address('TCP', '127.0.0.1', 80)
def setHost(self, host, port, ssl=0):
"""
Change the host and port the request thinks it's using.
@type host: C{bytes}
@param host: The value to which to change the host header.
@type ssl: C{bool}
@param ssl: A flag which, if C{True}, indicates that the request is
considered secure (if C{True}, L{isSecure} will return C{True}).
"""
self._forceSSL = ssl # set first so isSecure will work
if self.isSecure():
default = 443
else:
default = 80
if port == default:
hostHeader = host
else:
hostHeader = host + b":" + intToBytes(port)
self.requestHeaders.addRawHeader(b"host", hostHeader)
def getClient(self):
"""
Get the client's IP address, if it has one.
@return: The same value as C{getClientIP}.
@rtype: L{bytes}
"""
return self.getClientIP()
def redirect(self, url):
"""
Utility function that does a redirect.
The request should have finish() called after this.
"""
self.setResponseCode(FOUND)
self.setHeader(b"location", url)
class FakeRequest(object):
"""
A fake request suitable for use in place of C{twisted.web.http.Request}.
@param method: The HTTP method being invoked such as C{GET} or C{POST}.
@param uri: The URI of the resource being requested.
@param headers: A C{dict} of headers to send in the request.
@param args: A C{dict} of arguments that should be appended to the URI in
the request.
"""
content = None
def __init__(self, method, uri, headers=None, args=None):
self.method = method
self.uri = uri
self.requestHeaders = Headers(headers)
self.responseHeaders = Headers()
self.args = {'verb': [method]}
if args:
self.args.update(args)
self._fluidDB_reqid = 'xxx'
self.finished = False
def finish(self):
"""
Indicate that all response data has been written to this request.
"""
self.finished = True
def getHeader(self, key):
"""
Get the value of an HTTP request header.
@param key: The name of the HTTP header to retrieve a value for.
@return: The value set for the header or C{None} if a value is not
available.
"""
value = self.requestHeaders.getRawHeaders(key)
if value is not None:
return value[-1]
def getResponseHeader(self, key):
"""
Get the value of an HTTP response header.
@param key: The name of the HTTP header to retrieve a value for.
@return: The value set for the header or C{None} if a value is not
available.
"""
value = self.responseHeaders.getRawHeaders(key)
if value is not None:
return value[-1]
def setHeader(self, name, value):
"""
Set an HTTP header to include in the response returned to the client.
@param name: The name of the header to set.
@param value: The value to set.
"""
self.responseHeaders.setRawHeaders(name, [value])
def getAllHeaders(self):
"""
Return all the request headers.
@return: A C{dict} of all headers, with their values.
"""
headers = {}
for k, v in self.requestHeaders.getAllRawHeaders():
headers[k.lower()] = v[-1]
return headers
def setResponseCode(self, code):
"""
Set the HTTP response code to return to the client.
@param code: An HTTP response code as defined in C{twisted.web.http}.
"""
self.status = code
def notifyFinish(self):
"""
Return a C{twisted.internet.Deferred} that fires when the request
finishes or errors if the client disconnects. Note that this method
is needed as resource.py calls it, but we do not need to actually
fire the returned deferred for our test (which causes a synchronous
exception to immediately be returned to the request errback).
@return: C{twisted.internet.Deferred}
"""
return defer.succeed(None)
class DummyRequest(object):
"""
Represents a dummy or fake request. See L{twisted.web.server.Request}.
@ivar _finishedDeferreds: C{None} or a C{list} of L{Deferreds} which will
be called back with C{None} when C{finish} is called or which will be
errbacked if C{processingFailed} is called.
@type requestheaders: C{Headers}
@ivar requestheaders: A Headers instance that stores values for all request
headers.
@type responseHeaders: C{Headers}
@ivar responseHeaders: A Headers instance that stores values for all
response headers.
@type responseCode: C{int}
@ivar responseCode: The response code which was passed to
C{setResponseCode}.
@type written: C{list} of C{bytes}
@ivar written: The bytes which have been written to the request.
"""
uri = b'http://dummy/'
method = b'GET'
client = None
def registerProducer(self, prod,s):
self.go = 1
while self.go:
prod.resumeProducing()
def unregisterProducer(self):
self.go = 0
def __init__(self, postpath, session=None):
self.sitepath = []
self.written = []
self.finished = 0
self.postpath = postpath
self.prepath = []
self.session = None
self.protoSession = session or Session(0, self)
self.args = {}
self.requestHeaders = Headers()
self.responseHeaders = Headers()
self.responseCode = None
self._finishedDeferreds = []
self._serverName = b"dummy"
self.clientproto = b"HTTP/1.0"
def getAllHeaders(self):
"""
Return dictionary mapping the names of all received headers to the last
value received for each.
Since this method does not return all header information,
C{self.requestHeaders.getAllRawHeaders()} may be preferred.
NOTE: This function is a direct copy of
C{twisted.web.http.Request.getAllRawHeaders}.
"""
headers = {}
for k, v in self.requestHeaders.getAllRawHeaders():
headers[k.lower()] = v[-1]
return headers
def getHeader(self, name):
"""
Retrieve the value of a request header.
@type name: C{bytes}
@param name: The name of the request header for which to retrieve the
value. Header names are compared case-insensitively.
@rtype: C{bytes} or L{NoneType}
@return: The value of the specified request header.
"""
return self.requestHeaders.getRawHeaders(name.lower(), [None])[0]
def setHeader(self, name, value):
"""TODO: make this assert on write() if the header is content-length
"""
self.responseHeaders.addRawHeader(name, value)
def getSession(self):
if self.session:
return self.session
assert not self.written, "Session cannot be requested after data has been written."
self.session = self.protoSession
return self.session
def render(self, resource):
"""
Render the given resource as a response to this request.
This implementation only handles a few of the most common behaviors of
resources. It can handle a render method that returns a string or
C{NOT_DONE_YET}. It doesn't know anything about the semantics of
request methods (eg HEAD) nor how to set any particular headers.
Basically, it's largely broken, but sufficient for some tests at least.
It should B{not} be expanded to do all the same stuff L{Request} does.
Instead, L{DummyRequest} should be phased out and L{Request} (or some
other real code factored in a different way) used.
"""
result = resource.render(self)
if result is NOT_DONE_YET:
return
self.write(result)
self.finish()
def write(self, data):
if not isinstance(data, bytes):
raise TypeError("write() only accepts bytes")
self.written.append(data)
def notifyFinish(self):
"""
Return a L{Deferred} which is called back with C{None} when the request
is finished. This will probably only work if you haven't called
C{finish} yet.
"""
finished = Deferred()
self._finishedDeferreds.append(finished)
return finished
def finish(self):
"""
Record that the request is finished and callback and L{Deferred}s
waiting for notification of this.
"""
self.finished = self.finished + 1
if self._finishedDeferreds is not None:
observers = self._finishedDeferreds
self._finishedDeferreds = None
for obs in observers:
obs.callback(None)
def processingFailed(self, reason):
"""
Errback and L{Deferreds} waiting for finish notification.
"""
if self._finishedDeferreds is not None:
observers = self._finishedDeferreds
self._finishedDeferreds = None
for obs in observers:
obs.errback(reason)
def addArg(self, name, value):
self.args[name] = [value]
def setResponseCode(self, code, message=None):
"""
Set the HTTP status response code, but takes care that this is called
before any data is written.
"""
assert not self.written, "Response code cannot be set after data has been written: %s." % "@@@@".join(self.written)
self.responseCode = code
self.responseMessage = message
def setLastModified(self, when):
assert not self.written, "Last-Modified cannot be set after data has been written: %s." % "@@@@".join(self.written)
def setETag(self, tag):
assert not self.written, "ETag cannot be set after data has been written: %s." % "@@@@".join(self.written)
def getClientIP(self):
"""
Return the IPv4 address of the client which made this request, if there
is one, otherwise C{None}.
"""
if isinstance(self.client, IPv4Address):
return self.client.host
return None
def getRequestHostname(self):
"""
Get a dummy hostname associated to the HTTP request.
@rtype: C{bytes}
@returns: a dummy hostname
"""
return self._serverName
def getHost(self):
"""
Get a dummy transport's host.
@rtype: C{IPv4Address}
@returns: a dummy transport's host
"""
return IPv4Address('TCP', '127.0.0.1', 80)
def setHost(self, host, port, ssl=0):
"""
Change the host and port the request thinks it's using.
@type host: C{bytes}
@param host: The value to which to change the host header.
@type ssl: C{bool}
@param ssl: A flag which, if C{True}, indicates that the request is
considered secure (if C{True}, L{isSecure} will return C{True}).
"""
self._forceSSL = ssl # set first so isSecure will work
if self.isSecure():
default = 443
else:
default = 80
if port == default:
hostHeader = host
else:
hostHeader = host + b":" + intToBytes(port)
self.requestHeaders.addRawHeader(b"host", hostHeader)
def getClient(self):
"""
Get the client's IP address, if it has one.
@return: The same value as C{getClientIP}.
@rtype: L{bytes}
"""
return self.getClientIP()
def redirect(self, url):
"""
Utility function that does a redirect.
The request should have finish() called after this.
"""
self.setResponseCode(FOUND)
self.setHeader(b"location", url)
class FakeRequest(object):
"""
A fake C{twisted.web.http.Request} implementation, suitable for use in
tests.
"""
def __init__(self,
args=None,
postpath=None,
prepath=None,
path=None,
uri=None,
method='GET',
headers=None,
body=''):
self.args = {} if args is None else args
self.written = StringIO()
self.finished = False
self.code = None
self.method = method
self.path = path
self.postpath = postpath
self.prepath = prepath
self.requestHeaders = Headers({}) if headers is None else headers
self.responseHeaders = Headers({})
self.uri = uri
self._fluidDB_reqid = requestId()
self.content = StringIO(body)
def write(self, content):
"""Write data as a result of an HTTP request.
@param content: A C{str} containing the bytes to send as part of the
response body.
"""
if not isinstance(content, str):
raise RuntimeError('Only strings can be written.')
self.written.write(content)
def finish(self):
"""Indicate that all response data has been written to this request."""
if self.code is None:
self.code = 200
self.finished = True
def setResponseCode(self, code):
"""Set the HTTP response code.
@param code: An C{int} HTTP status code.
"""
self.code = code
def setHeader(self, key, value):
"""Set an HTTP response header.
@param key: The name of the response header.
@param value: The value for the response header.
"""
self.responseHeaders.setRawHeaders(key, [value])
def getHeader(self, key):
"""
Get a header from the request. This is copied from Twisted's
C{twisted.web.http.Request} class.
@param key: A C{str} indicating the header to return.
@return: The C{str} header value from the request if it exists,
else C{None}.
"""
value = self.requestHeaders.getRawHeaders(key)
if value is not None:
return value[-1]
def getResponseHeader(self, key):
"""Get a header from the response.
@param key: A C{str} indicating the header to return.
@return: The C{str} header value from the response if it exists,
else C{None}.
"""
value = self.responseHeaders.getRawHeaders(key)
if value is not None:
return value[-1]
@property
def response(self):
"""The HTTP response body."""
return self.written.getvalue()
def getAllHeaders(self):
"""
Get all the request headers. This is copied from Twisted's
C{twisted.web.http.Request} class.
@return: A C{dict} of request header name -> value.
"""
headers = {}
for k, v in self.requestHeaders.getAllRawHeaders():
headers[k.lower()] = v[-1]
return headers
def notifyFinish(self):
"""
Return a C{twisted.internet.Deferred} that fires when the request
finishes or errors if the client disconnects. Note that this method
is needed as resource.py calls it, but we do not need to actually
fire the returned deferred for our tests (which cause synchronous
exceptions to immediately be returned to the request errback).
@return: A C{Deferred} as just described.
"""
return succeed(None)
def isSecure(self):
"""Is the request secure?
@return: a C{bool} that is C{True} if the request is secure.
"""
# The way we tell if a request is secure is based on a header set
# for us by nginx. Our Twisted web service only handles plain http
# requests that are forwarded to it by nginx (via haproxy).
secureHeader = self.getHeader('X-Forwarded-Protocol')
return (secureHeader == 'https')
class FakeRequest(object):
"""
I pretend to be an HTTP request, with a handful of required methods
used by Twisted resource classes (e.g., write, finish,
setResponseCode).
"""
_fluidDB_reqid = None
body = ''
def __init__(self, method, d=None, headers=None, hostname=None):
"""
Initialize. d (if not None) is a deferred that will fire with the
body of the response when request.finish is called.
@param d: A C{Deferred} instance to fire when the request is finished.
@param headers: a C{dict} of headers and values for the request.
"""
self.method = method
self.d = d
self.uri = None
self.requestHeaders = Headers(headers)
self.responseHeaders = Headers()
self.args = {'verb': [method]}
self.hostname = hostname
self.content = StringIO.StringIO()
def isSecure(self):
"""
Mark the request as not being secure (i.e., not HTTPS). The
isSecure is used by the FluidDB wsfe to construct the Location
header in the response to a POST.
@return: A constant, C{False}, seeing as we don't care about
the return value, only that we provide the method.
"""
return False
def finish(self):
"""
Indicate that the request has been fully serviced. Send the
response body (as accumulated by self.write) back to the deferred
(if any) we were passed in __init__.
@return: C{None}.
"""
if self.d:
self.d.callback(self.body)
def write(self, data):
"""
Fake the writing of data back to the client. Instead, we
accumuluate it so as to deliver it to our test client when
self.finish is called.
@param data: A C{str} of response payload data.
@return: C{None}.
"""
self.body += data
def getHeader(self, key):
"""
Get a header from the request. This is copied from Twisted's
twisted.web.http.Request class.
@param key: A C{str} indicating the header to return.
@return: The C{str} header value from the request if it exists,
else C{None}.
"""
value = self.requestHeaders.getRawHeaders(key)
if value is not None:
return value[-1]
def getResponseHeader(self, key):
"""
Get a header from the response.
@param key: A C{str} indicating the header to return.
@return: The C{str} header value from the response if it exists,
else C{None}.
"""
value = self.responseHeaders.getRawHeaders(key)
if value is not None:
return value[-1]
def setHeader(self, name, value):
"""
Set a header for the HTTP response. This is copied from Twisted's
twisted.web.http.Request class.
@param name: A C{str} indicating the header to set.
@param value: A C{str} values for the header.
@return: C{None}.
"""
self.responseHeaders.setRawHeaders(name, [value])
def getAllHeaders(self):
"""
Get all the request headers. This is copied from Twisted's
twisted.web.http.Request class.
@return: A C{dict} of request header name -> value.
"""
headers = {}
for k, v in self.requestHeaders.getAllRawHeaders():
headers[k.lower()] = v[-1]
return headers
def setResponseCode(self, code):
"""
Set the response status code.
@param code: An HTTP status code.
@return: C{None}.
"""
self.status = code
def notifyFinish(self):
"""
Return a C{twisted.internet.Deferred} that fires when the request
finishes or errors if the client disconnects. Note that this method
is needed as resource.py calls it, but we do not need to actually
fire the returned deferred for our tests (which cause synchronous
exceptions to immediately be returned to the request errback).
@return: C{twisted.internet.Deferred}
"""
return defer.succeed(None)
def getRequestHostname(self):
"""
Return the hostname that was used for issuing this request.
"""
return self.hostname
class FakeRequest(object):
"""
A fake C{twisted.web.http.Request} implementation, suitable for use in
tests.
"""
def __init__(self, args=None, postpath=None, prepath=None, path=None,
uri=None, method='GET', headers=None, body=''):
self.args = {} if args is None else args
self.written = StringIO()
self.finished = False
self.code = None
self.method = method
self.path = path
self.postpath = postpath
self.prepath = prepath
self.requestHeaders = Headers({}) if headers is None else headers
self.responseHeaders = Headers({})
self.uri = uri
self._fluidDB_reqid = requestId()
self.content = StringIO(body)
def write(self, content):
"""Write data as a result of an HTTP request.
@param content: A C{str} containing the bytes to send as part of the
response body.
"""
if not isinstance(content, str):
raise RuntimeError('Only strings can be written.')
self.written.write(content)
def finish(self):
"""Indicate that all response data has been written to this request."""
if self.code is None:
self.code = 200
self.finished = True
def setResponseCode(self, code):
"""Set the HTTP response code.
@param code: An C{int} HTTP status code.
"""
self.code = code
def setHeader(self, key, value):
"""Set an HTTP response header.
@param key: The name of the response header.
@param value: The value for the response header.
"""
self.responseHeaders.setRawHeaders(key, [value])
def getHeader(self, key):
"""
Get a header from the request. This is copied from Twisted's
C{twisted.web.http.Request} class.
@param key: A C{str} indicating the header to return.
@return: The C{str} header value from the request if it exists,
else C{None}.
"""
value = self.requestHeaders.getRawHeaders(key)
if value is not None:
return value[-1]
def getResponseHeader(self, key):
"""Get a header from the response.
@param key: A C{str} indicating the header to return.
@return: The C{str} header value from the response if it exists,
else C{None}.
"""
value = self.responseHeaders.getRawHeaders(key)
if value is not None:
return value[-1]
@property
def response(self):
"""The HTTP response body."""
return self.written.getvalue()
def getAllHeaders(self):
"""
Get all the request headers. This is copied from Twisted's
C{twisted.web.http.Request} class.
@return: A C{dict} of request header name -> value.
"""
headers = {}
for k, v in self.requestHeaders.getAllRawHeaders():
headers[k.lower()] = v[-1]
return headers
def notifyFinish(self):
"""
Return a C{twisted.internet.Deferred} that fires when the request
finishes or errors if the client disconnects. Note that this method
is needed as resource.py calls it, but we do not need to actually
fire the returned deferred for our tests (which cause synchronous
exceptions to immediately be returned to the request errback).
@return: A C{Deferred} as just described.
"""
return succeed(None)
def isSecure(self):
"""Is the request secure?
@return: a C{bool} that is C{True} if the request is secure.
"""
# The way we tell if a request is secure is based on a header set
# for us by nginx. Our Twisted web service only handles plain http
# requests that are forwarded to it by nginx (via haproxy).
secureHeader = self.getHeader('X-Forwarded-Protocol')
return (secureHeader == 'https')
def request(self,
method,
path,
headers=None,
body=None,
expected_code=200,
retry=0,
delay=0):
"""
Send an http request to the gerrit service for the given path.
If 'retry' is specified, transient errors (http response code 500-599) will
be retried after an exponentially-increasing delay.
Returns a Deferred which will call back with the parsed json body of the
gerrit server's response.
Args:
method: 'GET', 'POST', etc.
path: Path element of the url.
headers: dict of http request headers.
body: json-encodable body of http request.
expected_code: http response code expected in reply.
retry: How many times to retry transient errors.
delay: Wait this many seconds before sending the request.
"""
retry_delay = delay * 2 if delay else 0.5
retry_args = (method, path, headers, body, expected_code, retry - 1,
retry_delay)
if not path.startswith('/'):
path = '/' + path
if not headers:
headers = Headers()
else:
# Make a copy so mutations don't affect retry attempts.
headers = Headers(dict(headers.getAllRawHeaders()))
if self.auth_token:
if not path.startswith('/a/'):
path = '/a' + path
headers.setRawHeaders('Authorization', [self.auth_token])
url = '%s://%s%s' % (self.gerrit_protocol, self.gerrit_host, path)
if body:
body = JsonBodyProducer(json.dumps(body))
headers.setRawHeaders('Content-Type', ['application/json'])
if DEBUG:
log.msg(url)
if delay:
d = defer.succeed(None)
d.addCallback(reactor.callLater, delay, Agent.request, self,
method, str(url), headers, body)
else:
d = Agent.request(self, method, str(url), headers, body)
def _check_code(response):
if response.code == expected_code:
return response
if retry > 0 and response.code >= 500 and response.code < 600:
return self.request(*retry_args)
msg = 'Failed gerrit request (code %s, expected %s): %s' % (
response.code, expected_code, url)
raise GerritError(msg, response.code)
d.addCallback(_check_code)
d.addCallback(JsonResponse.Get, url=url)
return d
class FakeRequest(object):
"""
I pretend to be an HTTP request, with a handful of required methods
used by Twisted resource classes (e.g., write, finish,
setResponseCode).
"""
_fluidDB_reqid = None
body = ''
def __init__(self, method, d=None, headers=None, hostname=None):
"""
Initialize. d (if not None) is a deferred that will fire with the
body of the response when request.finish is called.
@param d: A C{Deferred} instance to fire when the request is finished.
@param headers: a C{dict} of headers and values for the request.
"""
self.method = method
self.d = d
self.uri = None
self.requestHeaders = Headers(headers)
self.responseHeaders = Headers()
self.args = {'verb': [method]}
self.hostname = hostname
self.content = StringIO.StringIO()
def isSecure(self):
"""
Mark the request as not being secure (i.e., not HTTPS). The
isSecure is used by the FluidDB wsfe to construct the Location
header in the response to a POST.
@return: A constant, C{False}, seeing as we don't care about
the return value, only that we provide the method.
"""
return False
def finish(self):
"""
Indicate that the request has been fully serviced. Send the
response body (as accumulated by self.write) back to the deferred
(if any) we were passed in __init__.
@return: C{None}.
"""
if self.d:
self.d.callback(self.body)
def write(self, data):
"""
Fake the writing of data back to the client. Instead, we
accumuluate it so as to deliver it to our test client when
self.finish is called.
@param data: A C{str} of response payload data.
@return: C{None}.
"""
self.body += data
def getHeader(self, key):
"""
Get a header from the request. This is copied from Twisted's
twisted.web.http.Request class.
@param key: A C{str} indicating the header to return.
@return: The C{str} header value from the request if it exists,
else C{None}.
"""
value = self.requestHeaders.getRawHeaders(key)
if value is not None:
return value[-1]
def getResponseHeader(self, key):
"""
Get a header from the response.
@param key: A C{str} indicating the header to return.
@return: The C{str} header value from the response if it exists,
else C{None}.
"""
value = self.responseHeaders.getRawHeaders(key)
if value is not None:
return value[-1]
def setHeader(self, name, value):
"""
Set a header for the HTTP response. This is copied from Twisted's
twisted.web.http.Request class.
@param name: A C{str} indicating the header to set.
@param value: A C{str} values for the header.
@return: C{None}.
"""
self.responseHeaders.setRawHeaders(name, [value])
def getAllHeaders(self):
"""
Get all the request headers. This is copied from Twisted's
twisted.web.http.Request class.
@return: A C{dict} of request header name -> value.
"""
headers = {}
for k, v in self.requestHeaders.getAllRawHeaders():
headers[k.lower()] = v[-1]
return headers
def setResponseCode(self, code):
"""
Set the response status code.
@param code: An HTTP status code.
@return: C{None}.
"""
self.status = code
def notifyFinish(self):
"""
Return a C{twisted.internet.Deferred} that fires when the request
finishes or errors if the client disconnects. Note that this method
is needed as resource.py calls it, but we do not need to actually
fire the returned deferred for our tests (which cause synchronous
exceptions to immediately be returned to the request errback).
@return: C{twisted.internet.Deferred}
"""
return defer.succeed(None)
def getRequestHostname(self):
"""
Return the hostname that was used for issuing this request.
"""
return self.hostname
def quickHTTP(host, method, path):
headers = Headers()
headers = Headers(dict(headers.getAllRawHeaders()))
headers.addRawHeader('host', host)
return Request(method, path, headers, None)
