def test_http_response_early_failure(self):
header = ('the runtime process gave a bad HTTP response: '
'IncompleteRead(0 bytes read)\n\n')
def dave_message():
return "I'm sorry, Dave. I'm afraid I can't do that.\n"
self.proxy = http_proxy.HttpProxy(
host='localhost',
port=23456,
instance_died_unexpectedly=lambda: False,
instance_logs_getter=dave_message,
error_handler_file=None)
login.get_user_info(None).AndReturn(('', False, ''))
httplib.HTTPConnection.connect()
httplib.HTTPConnection.request(
'GET', '/get%20request?key=value', None, {
'HEADER': 'value',
http_runtime_constants.REQUEST_ID_HEADER: 'request id',
'X-AppEngine-Country': 'ZZ',
'X-Appengine-User-Email': '',
'X-Appengine-User-Id': '',
'X-Appengine-User-Is-Admin': '0',
'X-Appengine-User-Nickname': '',
'X-Appengine-User-Organization': '',
'X-APPENGINE-DEV-SCRIPT': 'get.py',
'X-APPENGINE-SERVER-NAME': 'localhost',
'X-APPENGINE-SERVER-PORT': '8080',
'X-APPENGINE-SERVER-PROTOCOL': 'HTTP/1.1',
})
httplib.HTTPConnection.getresponse().AndRaise(
httplib.IncompleteRead(''))
httplib.HTTPConnection.close()
environ = {
'HTTP_HEADER': 'value',
'PATH_INFO': '/get request',
'QUERY_STRING': 'key=value',
'HTTP_X_APPENGINE_USER_ID': '123',
'SERVER_NAME': 'localhost',
'SERVER_PORT': '8080',
'SERVER_PROTOCOL': 'HTTP/1.1',
}
self.mox.ReplayAll()
expected_headers = {
'Content-Type': 'text/plain',
'Content-Length': '%d' % (len(header) + len(dave_message()))
}
self.assertResponse('500 Internal Server Error',
expected_headers,
header + dave_message(),
self.proxy.handle,
environ,
url_map=self.url_map,
match=re.match(self.url_map.url, '/get%20request'),
request_id='request id',
request_type=instance.NORMAL_REQUEST)
self.mox.VerifyAll()
def _read_chunked(self, amt):
chunk_left = self.chunk_left
value = ''
# XXX This accumulates chunks by repeated string concatenation,
# which is not efficient as the number or size of chunks gets big.
while True:
if chunk_left is None:
line = self.fp.readline()
i = line.find(';')
if i >= 0:
line = line[:i] # strip chunk-extensions
try:
chunk_left = int(line, 16)
except ValueError:
# close the connection as protocol synchronisation is
# probably lost
self.close()
raise httplib.IncompleteRead(value)
if chunk_left == 0:
break
if amt is None:
value += self._safe_read(chunk_left)
elif amt < chunk_left:
value += self._safe_read(amt)
self.chunk_left = chunk_left - amt
return value
elif amt == chunk_left:
value += self._safe_read(amt)
self._safe_read(2) # toss the CRLF at the end of the chunk
self.chunk_left = None
return value
else:
value += self._safe_read(chunk_left)
amt -= chunk_left
# we read the whole chunk, get another
self._safe_read(2) # toss the CRLF at the end of the chunk
chunk_left = None
# read and discard trailer up to the CRLF terminator
### note: we shouldn't have any trailers!
while True:
line = self.fp.readline()
if not line:
# a vanishingly small number of sites EOF without
# sending the trailer
break
if line == '\r\n':
break
# we read everything; close the "file"
self.close()
return value
def test_http_response_late_failure(self):
line0 = "I know I've made some very poor decisions recently...\n"
def dave_message():
return "I'm afraid. I'm afraid, Dave.\n"
self.proxy = http_proxy.HttpProxy(
host='localhost',
port=23456,
instance_died_unexpectedly=lambda: False,
instance_logs_getter=dave_message,
error_handler_file=None)
response = FakeHttpResponse(200, 'OK', [], line0)
response.partial_read_error = httplib.IncompleteRead('')
login.get_user_info(None).AndReturn(('', False, ''))
httplib.HTTPConnection.connect()
httplib.HTTPConnection.request(
'GET', '/get%20request?key=value', None, {
'HEADER': 'value',
http_runtime_constants.REQUEST_ID_HEADER: 'request id',
'X-AppEngine-Country': 'ZZ',
'X-Appengine-User-Email': '',
'X-Appengine-User-Id': '',
'X-Appengine-User-Is-Admin': '0',
'X-Appengine-User-Nickname': '',
'X-Appengine-User-Organization': '',
'X-APPENGINE-DEV-SCRIPT': 'get.py',
'X-APPENGINE-SERVER-NAME': 'localhost',
'X-APPENGINE-SERVER-PORT': '8080',
'X-APPENGINE-SERVER-PROTOCOL': 'HTTP/1.1',
})
httplib.HTTPConnection.getresponse().AndReturn(response)
httplib.HTTPConnection.close()
environ = {
'HTTP_HEADER': 'value',
'PATH_INFO': '/get request',
'QUERY_STRING': 'key=value',
'HTTP_X_APPENGINE_USER_ID': '123',
'SERVER_NAME': 'localhost',
'SERVER_PORT': '8080',
'SERVER_PROTOCOL': 'HTTP/1.1',
}
self.mox.ReplayAll()
self.assertResponse('200 OK', {},
line0,
self.proxy.handle,
environ,
url_map=self.url_map,
match=re.match(self.url_map.url, '/get%20request'),
request_id='request id',
request_type=instance.NORMAL_REQUEST)
self.mox.VerifyAll()
def _update_chunk_length(self):
# First, we'll figure out length of a chunk and then
# we'll try to read it from socket.
if self.chunk_left is not None:
return
line = self._fp.fp.readline()
line = line.split(';', 1)[0]
try:
self.chunk_left = int(line, 16)
except ValueError:
# Invalid chunked protocol response, abort.
self.close()
raise httplib.IncompleteRead(line)
def test_http_response_early_failure(self):
header = ('the runtime process gave a bad HTTP response: '
'IncompleteRead(0 bytes read)\n\n')
stderr0 = "I'm sorry, Dave. I'm afraid I can't do that.\n"
self.proxy._stderr_tee = FakeTee(stderr0)
login.get_user_info(None).AndReturn(('', False, ''))
httplib.HTTPConnection.connect()
httplib.HTTPConnection.request(
'GET', '/get%20request?key=value', '', {
'HEADER': 'value',
http_runtime_constants.REQUEST_ID_HEADER: 'request id',
'X-AppEngine-Country': 'ZZ',
'X-Appengine-Internal-User-Email': '',
'X-Appengine-Internal-User-Id': '',
'X-Appengine-Internal-User-Is-Admin': '0',
'X-Appengine-Internal-User-Nickname': '',
'X-Appengine-Internal-User-Organization': '',
'X-APPENGINE-INTERNAL-SCRIPT': 'get.py',
'X-APPENGINE-INTERNAL-SERVER-NAME': 'localhost',
'X-APPENGINE-INTERNAL-SERVER-PORT': '8080',
'X-APPENGINE-INTERNAL-SERVER-PROTOCOL': 'HTTP/1.1',
})
httplib.HTTPConnection.getresponse().AndRaise(
httplib.IncompleteRead(''))
httplib.HTTPConnection.close()
environ = {
'HTTP_HEADER': 'value',
'PATH_INFO': '/get request',
'QUERY_STRING': 'key=value',
'HTTP_X_APPENGINE_INTERNAL_USER_ID': '123',
'SERVER_NAME': 'localhost',
'SERVER_PORT': '8080',
'SERVER_PROTOCOL': 'HTTP/1.1',
}
self.mox.ReplayAll()
expected_headers = {
'Content-Type': 'text/plain',
'Content-Length': '121', #str(len(header) + len(stderr0)),
}
self.assertResponse('500 Internal Server Error',
expected_headers,
header + stderr0,
self.proxy.handle,
environ,
url_map=self.url_map,
match=re.match(self.url_map.url, '/get%20request'),
request_id='request id',
request_type=instance.NORMAL_REQUEST)
self.mox.VerifyAll()
def read_chunks(self):
"""Return the response body content and timing data.
The returned chunks have the chunk size and CRLFs stripped off.
If the response was compressed, the returned data is still compressed.
Returns:
(chunks, delays)
chunks:
[response_body] # non-chunked responses
[chunk_1, chunk_2, ...] # chunked responses
delays:
[0] # non-chunked responses
[chunk_1_first_byte_delay, ...] # chunked responses
The delay for the first body item should be recorded by the caller.
"""
buf = []
chunks = []
delays = []
if not self.chunked:
chunks.append(self.read())
delays.append(0)
else:
start = TIMER()
try:
while True:
line = self.fp.readline()
chunk_size = self._read_chunk_size(line)
if chunk_size is None:
raise httplib.IncompleteRead(''.join(chunks))
if chunk_size == 0:
break
delays.append(TIMER() - start)
chunks.append(self._safe_read(chunk_size))
self._safe_read(2) # skip the CRLF at the end of the chunk
start = TIMER()
# Ignore any trailers.
while True:
line = self.fp.readline()
if not line or line == '\r\n':
break
finally:
self.close()
return chunks, delays
def test_http_response_late_failure(self):
line0 = "I know I've made some very poor decisions recently...\n"
line1 = "I'm afraid. I'm afraid, Dave.\n"
line2 = "Dave, my mind is going. I can feel it.\n"
response = FakeHttpResponse(200, 'OK', [], line0)
response.partial_read_error = httplib.IncompleteRead('')
self.proxy._stderr_tee = FakeTee(line1 + line2)
login.get_user_info(None).AndReturn(('', False, ''))
httplib.HTTPConnection.connect()
httplib.HTTPConnection.request(
'GET', '/get%20request?key=value', '', {
'HEADER': 'value',
http_runtime_constants.REQUEST_ID_HEADER: 'request id',
'X-AppEngine-Country': 'ZZ',
'X-Appengine-Internal-User-Email': '',
'X-Appengine-Internal-User-Id': '',
'X-Appengine-Internal-User-Is-Admin': '0',
'X-Appengine-Internal-User-Nickname': '',
'X-Appengine-Internal-User-Organization': '',
'X-APPENGINE-INTERNAL-SCRIPT': 'get.py',
'X-APPENGINE-INTERNAL-SERVER-NAME': 'localhost',
'X-APPENGINE-INTERNAL-SERVER-PORT': '8080',
'X-APPENGINE-INTERNAL-SERVER-PROTOCOL': 'HTTP/1.1',
})
httplib.HTTPConnection.getresponse().AndReturn(response)
httplib.HTTPConnection.close()
environ = {
'HTTP_HEADER': 'value',
'PATH_INFO': '/get request',
'QUERY_STRING': 'key=value',
'HTTP_X_APPENGINE_INTERNAL_USER_ID': '123',
'SERVER_NAME': 'localhost',
'SERVER_PORT': '8080',
'SERVER_PROTOCOL': 'HTTP/1.1',
}
self.mox.ReplayAll()
self.assertResponse('200 OK', {},
line0,
self.proxy.handle,
environ,
url_map=self.url_map,
match=re.match(self.url_map.url, '/get%20request'),
request_id='request id',
request_type=instance.NORMAL_REQUEST)
self.mox.VerifyAll()
def upload_cloudinary(user_id, filename):
if not user_id:
raise httplib.CannotSendRequest(
"Cloudinary upload called with no user_id.")
if not filename:
raise httplib.CannotSendRequest(
"Cloudinary upload called with no filename.")
req_url = "{}/{}/{}".format(CLOUDINARY_METHOD, user_id, filename)
try:
# Use HEAD since no data is required
httpConn.request("HEAD", req_url)
except Exception as e:
raise httplib.CannotSendRequest(
"Cloudinary request failed for {}.".format(req_url))
cloudinary_return = False
cloudinary_attempts = 0
# Retry Cloudinary trigger until success or at most 3 times
while not cloudinary_return and cloudinary_attempts < 3:
cloudinary_attempts += 1
try:
resp = httpConn.getresponse()
if resp.status != 200:
# Cloudinary failed, return from method
raise httplib.BadStatusLine(
"Cloudinary HTTP HEAD status {} reason {} for {}.".format(
resp.status, resp.reason, req_url))
else:
# Read the response to enabled next request
resp.read()
cloudinary_return = True
except httplib.ResponseNotReady as e:
logger.info(
"Cloudinary HTTP response for {} not ready after attempt {}, retrying..."
.format(req_url, cloudinary_attempts))
# The request never returned
if not cloudinary_return:
raise httplib.IncompleteRead(
"Cloudinary HTTP request for {} never returned.".format(req_url))
def _read_chunked(self,amt):
if not(self.read_by_chunks):
return httplib.HTTPResponse._read_chunked(self,amt)
assert self.chunked != httplib._UNKNOWN
line = self.fp.readline(httplib._MAXLINE + 1)
if len(line) > httplib._MAXLINE:
raise httplib.LineTooLong("chunk size")
i = line.find(';')
if i >= 0:
line = line[:i]
try:
chunk_left = int(line, 16)
except ValueError:
self.close()
raise httplib.IncompleteRead('')
if chunk_left == 0:
return ''
ret = self._safe_read(chunk_left)
self._safe_read(2)
return ret
def read(self, chunksize=None):
request = get_current_browser_request()
timeline = get_request_timeline(request)
action = timeline.start("librarian-read", self.url)
try:
if chunksize is None:
s = self.file.read()
else:
if self.length is not None:
chunksize = min(chunksize, self.length)
s = self.file.read(chunksize)
if self.length is not None:
self.length -= len(s)
# httplib doesn't quite do all the checking we need: in
# particular, it won't fail on a short bounded-size read
# from a non-chunked-transfer-coding resource. Check this
# manually.
if not s and chunksize != 0 and self.length:
raise httplib.IncompleteRead(s, expected=self.length)
return s
finally:
action.finish()
def _read_chunk(self):
# This should always occur on a chunk boundry, so we should never
# need to store a lot of text in the buffer.
text = ''
chunk_size = None
if len(self._buffer) < 32:
ready, d1, d2 = select.select([self._socket], [], [])
if len(ready) == 0:
raise httplib.NotConnected()
text = self._socket.recv(32)
if len(text) == 0:
self._socket.close()
raise httplib.NotConnected()
self._buffer += text
eol = self._buffer.find('\r\n')
if eol >= 0:
line = self._buffer[:eol]
i = line.find(';')
if i >= 0:
line = line[:i] # strip chunk-extensions
try:
chunk_size = int(line, 16) + 2
except ValueError:
# close the connection as protocol synchronisation is
# probably lost
self._socket.close()
raise httplib.IncompleteRead(line)
self._buffer = self._buffer[(eol + 2):]
chunk = self._read_all(chunk_size)
self._buffer = chunk[chunk_size:]
return chunk[:(chunk_size - 2)]
def __call__(self, req_url):
if self._num_failures <= 0:
return self._canned_response
self._num_failures = self._num_failures - 1
raise httplib.IncompleteRead("incomplete read!")
