def test_noMessageInvalidStatus(self):
"""
If no C{message} argument is passed to the L{Error} constructor and
C{code} isn't a valid HTTP status code, C{message} stays C{None}.
"""
e = error.Error("InvalidCode")
self.assertEquals(e.message, None)
def test_messageExists(self):
"""
If a C{message} argument is passed to the L{Error} constructor, the
C{message} isn't affected by the value of C{status}.
"""
e = error.Error(b"200", b"My own message")
self.assertEqual(e.message, b"My own message")
def handleResponse(self, response):
if self.quietLoss:
return
if self.failed:
self.factory.noPage(
failure.Failure(
error.Error(
self.status, self.message, response)))
elif self.length != None and self.length != 0:
self.factory.noPage(failure.Failure(
client.PartialDownloadError(self.status, self.message, response)))
else:
if self.decode:
s = StringIO()
s.write(response)
s.seek(-1)
g = GzipFile(fileobj=s, mode='rb')
try:
response = g.read()
except IOError:
self.factory.noPage(failure.Failure(
client.PartialDownloadError(self.status, self.message, response)))
self.transport.loseConnection()
return
g.close()
self.factory.page(response)
# server might be stupid and not close connection.
self.transport.loseConnection()
def test_noMessageValidStatus(self):
"""
If no C{message} argument is passed to the L{Error} constructor and the
C{code} argument is a valid HTTP status code, C{code} is mapped to a
descriptive string to which C{message} is assigned.
"""
e = error.Error(b"200")
self.assertEqual(e.message, b"OK")
def handleResponseEnd(self):
if self.transmittingPage:
self.factory.pageEnd()
self.transmittingPage = 0
if self.failed:
self.factory.noPage(
failure.Failure(error.Error(self.status, self.message, None)))
self.transport.loseConnection()
def retrieve(self, request, *args, **kwargs):
try:
_filter = request.args.get('.filter', 'default')
serializer = self.get_serializer_class()(fields_filter=_filter)
self.write_headers(request)
result = yield serializer.get(self.get_object_id())
defer.returnValue(json.dumps(result, default=json_default))
except NotFound:
raise web_error.Error(http_status.HTTP_404_NOT_FOUND)
def cb_process_resp(body, response):
# Emulate HTTPClientFactory and raise t.w.e.Error
# and PageRedirect if we have errors.
if response.code > 299 and response.code < 400:
raise tw_error.PageRedirect(response.code, body)
elif response.code > 399:
raise tw_error.Error(response.code, body)
return body
def _process_response(_, response):
# this sets ctx.in_error if there's an error, and ctx.in_object if
# there's none.
self.get_in_object(self.ctx)
if self.ctx.in_error is not None:
raise self.ctx.in_error
elif response.code >= 400:
raise werror.Error(response.code)
return self.ctx.in_object
def cb_process_resp(body, response):
# twisted.web.error imports reactor
from twisted.web import error as tw_error
# Emulate HTTPClientFactory and raise t.w.e.Error
# and PageRedirect if we have errors.
if response.code > 299 and response.code < 400:
raise tw_error.PageRedirect(response.code, body)
elif response.code > 399:
raise tw_error.Error(response.code, body)
return body
def handleResponse(self, response):
if self.status != '200':
err = error.Error(self.status, self.message, response)
self.factory.onFailure(failure.Failure(err))
else:
try:
config = json.loads(response)
self.factory.onUpdate(config, self.etcdIndex)
except Exception:
self.factory.onFailure(failure.Failure())
self.transport.loseConnection()
def handleResponse(self, response):
log.debug("%s response: %s" % (self.status, response),
system="config-etcd")
if self.status != '200':
err = error.Error(self.status, self.message, response)
self.factory.onFailure(failure.Failure(err))
elif response is not None and len(response):
try:
config = json.loads(response)
self.factory.onUpdate(config, self.etcdIndex)
except Exception, err:
msg = "Error: %s Etcd response: %s" % (err, response)
self.factory.onFailure(msg)
def go(self):
"""
Attempt to download L{self.url}.
"""
headers = self.kwargs.pop('headers', Headers())
headers.setRawHeaders('user-agent', ['Eridanus IRC bot'])
response = yield treq.get(str(self.url),
timeout=self.timeout,
headers=headers,
*self.args,
**self.kwargs)
data = yield response.content()
if response.code // 100 == 2:
returnValue((data, response.headers))
else:
raise weberror.Error(response.code, response.phrase, data)
def test_response_receiver(self):
finished = defer.Deferred()
recv = ResponseReceiver(finished)
recv.dataReceived('bytes')
recv.dataReceived(' go')
recv.dataReceived('here.')
err = error.Error('Something Happened')
recv.connectionLost(Failure(err))
def checkError(result):
result.trap(error.Error)
self.assertRaises(error.Error, result.raiseException)
finished.addBoth(checkError)
return finished
def handleStatus(self, version, status, message):
""" Called when the status header is received """
if status == '404':
self.transport.loseConnection()
self.factory.error(failure.Failure(error.Error(http.NOT_FOUND)))
return
length = len(version) + len(status) + len(message)
if length > 50:
self.transport.loseConnection()
self.factory.error(
failure.Failure(
RTSPStatusError(
'Length of status message was too long: %s' % length,
version)))
return
print('Status: %s %s %s' % (status, message, version))
def handleResponse(self, response):
if self.quietLoss:
return
if self.failed:
self.factory.noPage(
failure.Failure(
error.Error(self.status, self.message, response)))
elif self.length != None and self.length != 0:
self.factory.noPage(
failure.Failure(
PartialDownloadError(self.status, self.message, response)))
else:
self.factory.page(response)
# server might be stupid and not close connection. admittedly
# the fact we do only one request per connection is also
# stupid...
self.transport.loseConnection()
def list(self, request, *args, **kwargs):
_args = regroup_args(request.args)
_args.update(kwargs)
fields_filter = _args.pop('.filter', 'default')
serializer = self.get_serializer_class()(fields_filter=fields_filter)
self.write_headers(request)
limit = _args.pop('.limit', settings.PAGE_SIZE)
offset = _args.pop('.offset', 0)
sort = make_sort_filter(_args.pop('.sort', None))
try:
result = yield serializer.find(_args,
skip=offset,
limit=limit,
sort=sort)
defer.returnValue(json.dumps(result, default=json_default))
except ValueError, ex:
raise web_error.Error(http_status.HTTP_406_NOT_ACCEPTABLE,
ex.message)
class FakeCurrencyClient:
def __init__(self):
self.serial = 1
self.check_note_result = True
self.commit_result = True
def message(self, string):
print "FakeCurrencyClient: " + string
def breakNote(self, (url, serial, name, value), *values):
if Verbose: self.message("breakNote vaues %s" % str(values))
if values:
values = map(int, values)
values.sort()
values.reverse()
notes = []
if values[-1] == 0:
notes.append((url, serial, name, value))
notes.append((url, 0, '', 0))
else:
for note_value in values:
if value < note_value: continue
count = value / note_value
value %= note_value
for i in xrange(count):
notes.append(
(url, self.serial, "%040d" % self.serial, note_value))
self.serial += 1
if value <= 0: break
if value > 0:
notes.append(
(url, self.serial, "%040d" % self.serial, note_value))
self.serial += 1
d = defer.Deferred()
if FakeCurrencyFailure:
reactor.callLater(
0, lambda: d.errback(
failure.Failure(
error.Error(500, "breakNote: fake error",
"(page content)"))))
else:
reactor.callLater(0, lambda: d.callback(notes))
return d
def test_connectHTTPError(self):
"""
HTTP errors cause reconnects after a delay with back-offs.
"""
self.setUpState('connecting')
callCount = 1
for delay in (10, 20, 40, 80, 160, 240, 240):
# Fail the connection attempt with a ConnectError
self.api.connectFail(http_error.Error(401))
self.clock.advance(0)
# The error is logged
self.assertEquals(1, len(self.flushLoggedErrors(http_error.Error)))
# A reconnect is done after the delay
self.clock.advance(delay)
callCount += 1
self.assertEqual(callCount, len(self.api.filterCalls))
class FakeCurrencyClient:
log = log.get_child('FakeCurrencyClient')
def __init__(self):
self.log = FakeCurrencyClient.log.get_instance(self, refs=[
('Currency', self, lambda currency: currency.serial)
])
self.serial = 1
self.check_note_result = True
self.commit_result = True
def breakNote(self, (url, serial, name, value), *values):
self.log.debug("breakNote values %s", values)
if values:
values = map(int, values)
values.sort()
values.reverse()
notes = []
if values[-1] == 0:
notes.append((url, serial, name, value))
notes.append((url, 0, '', 0))
else:
for note_value in values:
if value < note_value: continue
count = value / note_value
value %= note_value
for i in xrange(count):
notes.append((url, self.serial, "%040d" % self.serial, note_value))
self.serial += 1
if value <= 0: break
if value > 0:
notes.append((url, self.serial, "%040d" % self.serial, note_value))
self.serial += 1
d = defer.Deferred()
if FakeCurrencyFailure:
reactor.callLater(0, lambda: d.errback(failure.Failure(error.Error(500, "breakNote: fake error", "(page content)"))))
else:
reactor.callLater(0, lambda: d.callback(notes))
return d
def handleResponse(self, response):
if self.quietLoss:
return
if self.failed:
self.factory.noPage(
failure.Failure(
error.Error(self.status, self.message, response)))
if self.factory.method == 'HEAD':
# Callback with empty string, since there is never a response
# body for HEAD requests.
self.factory.page('')
elif self.length != None and self.length != 0:
self.factory.noPage(
failure.Failure(
PartialDownloadError(self.status, self.message, response)))
else:
self.factory.page(response)
# server might be stupid and not close connection. admittedly
# the fact we do only one request per connection is also
# stupid...
self.transport.loseConnection()
def handleResponse(self, response):
if self.quietLoss:
return
if self.failed:
self.factory.noPage(
failure.Failure(
error.Error(
self.status, self.message, response)))
elif self.factory.method != 'HEAD' and self.length != None and self.length != 0:
self.factory.noPage(failure.Failure(
client.PartialDownloadError(self.status, self.message, response)))
else:
if(self.headers.has_key('transfer-encoding') and
self.headers['transfer-encoding'][0].lower() == 'chunked'):
self.factory.page(de_chunk_payload(response))
else:
self.factory.page(response)
# server might be stupid and not close connection. admittedly
# the fact we do only one request per connection is also
# stupid...
self.quietLoss = 1
self.transport.loseConnection()
def handle_response(response, retries_remaining, next_delay):
if response.code in expected_code:
return response
error = twe.Error(
response.code,
message=
"Response status (%s) didn't match expected (%s) for '%s'" %
(response.code, expected_code, url))
log.msg(error.message)
# Do we retry?
if (retries_remaining > 0
and response.code >= httplib.INTERNAL_SERVER_ERROR):
log.msg(
"Query '%s' encountered transient error (%d); retrying "
"(%d remaining) in %s second(s)..." %
(url, response.code, retries_remaining, next_delay))
if (body_producer is not None
and IReusableBodyProducer.providedBy(body_producer)):
body_producer.reset()
d = deferLater(reactor, next_delay, do_request)
d.addCallback(handle_response, (retries_remaining - 1),
(next_delay * 2))
return d
# No retries remaining; return an error
if error_protocol is None:
raise error
d = error_protocol(response)
def cbRaiseError(response, error):
error.response = response
raise error
d.addCallback(cbRaiseError, error)
return d
def _check_error(resp):
if resp.code != 200:
raise web_error.Error(resp.code, resp.phrase)
return resp
def __init__(self, reason):
error.Error(self, reason)
self.reason = reason
def request(self,
method,
path,
headers=None,
body_producer=None,
expected_code=httplib.OK,
retry=None,
delay=None,
error_protocol=None):
"""
Constructs and initiates a HTTP request.
If supplied, 'error_protocol' is a function that will be used to convert
the Agent response into a string. In the event that an HTTP error is
encountered, the body of that HTTP response will be loaded through
'error_protocol'. The resulting 'twisted.web.error.Error' will then have
its 'response' field populated with the resulting value.
'error_protocol' is a function that accepts a 'Response' object and
returns a Deferred whose return value is the loaded body. Some examples of
such functions are:
- StringResponse.Get
- JsonResponse.Get
(error_protocol) Args:
response: (twisted.web.client.Response) the Response object to load
(error_protocol) Returns: (Deferred) A Deferred whose return value will be
loaded into the 'twisted.web.error.Error' that the error failure wraps.
Args:
method: (str) The HTTP request type (GET, PUT, POST, DELETE)
path: (str) The path within the Agent's host to query
headers: (Headers) If supplied, the starting HTTP Headers object
body_producer: (IBodyProducer) If supplied, the 'IBodyProducer' instance
that will be used to produce HTTP request's body; if None, the body
will be of length '0'.
expected_code: (int or tuple of ints) HTTP response code expected in
reply.
retry: (int) If non-zero, the number of times to retry when a transient
error is encountered. If None, the default 'retry' value will be
used.
delay: (int) The number of seconds of the initial retry delay; subsequent
delays will double this value. If 'None', the default delay will be
used. If no default delay was supplied either, a delay of 0.5 seconds
will be used if retrying.
error_protocol: (func) The function to use to load the HTTP response when
an error occurs; if this is 'None', the body will not be read on
error.
Returns: (Deferred) A deferred that will be invoked with a
'twisted.web.client.Response' instance.
"""
assert method in ('GET', 'PUT', 'POST', 'DELETE')
if (not isinstance(expected_code, tuple)
and not isinstance(expected_code, list)):
expected_code = (expected_code, )
headers = CloneHeaders(headers) # Handles 'None' case
url = self._buildRequest(path, headers)
# Get parameters, falling back on global values
if retry is None:
retry = self._retry_count
if delay is None:
delay = self._retry_delay
if body_producer is not None:
# If 'body_producer' supplies its own MIME type, use that.
if IMIMEBodyProducer.providedBy(body_producer):
headers.setRawHeaders('Content-Type',
[body_producer.getMIMEType()])
if self.verbose:
log.msg('%r %r' % (method, url))
for key, vlist in headers.getAllRawHeaders():
for val in vlist:
if key.lower() == 'authorization':
val = 'HIDDEN'
log.msg('(Header) %r: %r' % (key, val))
# Special case (StringBodyProducer): dump body if we can
if isinstance(body_producer, StringBodyProducer):
log.msg(body_producer.body_str)
if (self.read_only) and (method != 'GET'):
raise twe.Error(
httplib.METHOD_NOT_ALLOWED,
"Refusing to execute '%s' request to read-only Agent: %s" %
(method, url),
)
#
# Request/Retry Deferred Loop
#
# This Deferred loop starts wuith 'do_request' => 'handle_response'; if
# this encounters a transient error, 'handle_response' will enqueue another
# 'do_request' after a suitable delay until our retries are exhausted
#
def do_request():
return twisted.web.client.Agent.request(self, method, str(url),
headers, body_producer)
def handle_response(response, retries_remaining, next_delay):
if response.code in expected_code:
return response
error = twe.Error(
response.code,
message=
"Response status (%s) didn't match expected (%s) for '%s'" %
(response.code, expected_code, url))
log.msg(error.message)
# Do we retry?
if (retries_remaining > 0
and response.code >= httplib.INTERNAL_SERVER_ERROR):
log.msg(
"Query '%s' encountered transient error (%d); retrying "
"(%d remaining) in %s second(s)..." %
(url, response.code, retries_remaining, next_delay))
if (body_producer is not None
and IReusableBodyProducer.providedBy(body_producer)):
body_producer.reset()
d = deferLater(reactor, next_delay, do_request)
d.addCallback(handle_response, (retries_remaining - 1),
(next_delay * 2))
return d
# No retries remaining; return an error
if error_protocol is None:
raise error
d = error_protocol(response)
def cbRaiseError(response, error):
error.response = response
raise error
d.addCallback(cbRaiseError, error)
return d
# Make initial request
d = do_request()
d.addCallback(handle_response, retry, delay)
return d
def handleErrorResponse(self, response):
err = error.Error(self.status, self.message, response)
self.factory.onFailure(failure.Failure(err))
def request(self,
method,
path,
headers=None,
body_producer=None,
expected_code=httplib.OK,
retry=None,
delay=None,
error_protocol=None,
timeout=None):
"""
Constructs and initiates a HTTP request.
If supplied, 'error_protocol' is a function that will be used to convert
the Agent response into a string. In the event that an HTTP error is
encountered, the body of that HTTP response will be loaded through
'error_protocol'. The resulting 'twisted.web.error.Error' will then have
its 'response' field populated with the resulting value.
'error_protocol' is a function that accepts a 'Response' object and
returns a Deferred whose return value is the loaded body. Some examples of
such functions are:
- StringResponse.Get
- JsonResponse.Get
(error_protocol) Args:
response: (twisted.web.client.Response) the Response object to load
(error_protocol) Returns: (Deferred) A Deferred whose return value will be
loaded into the 'twisted.web.error.Error' that the error failure wraps.
Args:
method: (str) The HTTP request type (GET, PUT, POST, DELETE)
path: (str) The path within the Agent's host to query
headers: (Headers) If supplied, the starting HTTP Headers object
body_producer: (IBodyProducer) If supplied, the 'IBodyProducer' instance
that will be used to produce HTTP request's body; if None, the body
will be of length '0'.
expected_code: (int or tuple of ints) HTTP response code expected in
reply.
retry: (int) If non-zero, the number of times to retry when a transient
error is encountered. If None, the default 'retry' value will be
used.
delay: (number) The number of seconds of the initial retry delay;
subsequent delays will double this value. If 'None', the default delay
will be used. If no default delay was supplied either, a delay of 0.5
seconds will be used if retrying.
error_protocol: (func) The function to use to load the HTTP response when
an error occurs; if this is 'None', the body will not be read on
error.
timeout (number): if not None, the number of seconds to wait before
terminating the request. If a timeout is encountered, a TimeoutError
will be raised.
Returns: (Deferred) A deferred that will be invoked with a
'twisted.web.client.Response' instance.
"""
assert method in ('GET', 'PUT', 'POST', 'DELETE')
if (not isinstance(expected_code, tuple)
and not isinstance(expected_code, list)):
expected_code = (expected_code, )
headers = CloneHeaders(headers) # Handles 'None' case
url = self._buildRequest(path, headers)
# Get parameters, falling back on global values
retry = retry or self._retry_count
delay = (delay) if delay is not None else (self._retry_delay)
if body_producer is not None:
# If 'body_producer' supplies its own MIME type, use that.
if IMIMEBodyProducer.providedBy(body_producer):
headers.setRawHeaders('Content-Type',
[body_producer.getMIMEType()])
if self.verbose:
log.msg('%r %r' % (method, url))
for key, vlist in headers.getAllRawHeaders():
for val in vlist:
if key.lower() == 'authorization':
val = 'HIDDEN'
log.msg('(Header) %r: %r' % (key, val))
# Special case (StringBodyProducer): dump body if we can
if isinstance(body_producer, StringBodyProducer):
log.msg(body_producer.body_str)
if (self.read_only) and (method != 'GET'):
raise twe.Error(
httplib.METHOD_NOT_ALLOWED,
"Refusing to execute '%s' request to read-only Agent: %s" %
(method, url),
)
# Request/Retry Deferred Loop
while True:
# Make the request.
response = yield self._timeout_request(timeout, method, str(url),
headers, body_producer)
if response.code in expected_code:
defer.returnValue(response)
# The operation failed.
retry -= 1
error = twe.Error(
response.code,
message=
"Response status (%s) didn't match expected (%s) for '%s'" %
(response.code, expected_code, url))
log.msg(error.message)
# If we have no more retries, or if this was a non-transient failure,
# exit immediately with Twisted web error.
if retry <= 0 or response.code < httplib.INTERNAL_SERVER_ERROR:
# No more retries, return our error.
if error_protocol:
error.response = yield error_protocol(response)
raise error
# Retry after an exponentially-increasing delay.
log.msg("Query '%s' encountered transient error (%d); retrying "
"(%d remaining) in %s second(s)..." %
(url, response.code, retry, delay))
if (body_producer is not None
and IReusableBodyProducer.providedBy(body_producer)):
body_producer.reset()
yield self._deferred_sleep(delay)
delay *= 2
def makeError(dummy):
data = self.path.sibling('twitter_error.xml').open().read()
raise weberror.Error(404, None, data)
