def test_http_response_early_failure(self): header = ('the runtime process gave a bad HTTP response: ' 'IncompleteRead(0 bytes read)\n\n') def dave_message(): return "I'm sorry, Dave. I'm afraid I can't do that.\n" self.proxy = http_proxy.HttpProxy( host='localhost', port=23456, instance_died_unexpectedly=lambda: False, instance_logs_getter=dave_message, error_handler_file=None) login.get_user_info(None).AndReturn(('', False, '')) httplib.HTTPConnection.connect() httplib.HTTPConnection.request( 'GET', '/get%20request?key=value', None, { 'HEADER': 'value', http_runtime_constants.REQUEST_ID_HEADER: 'request id', 'X-AppEngine-Country': 'ZZ', 'X-Appengine-User-Email': '', 'X-Appengine-User-Id': '', 'X-Appengine-User-Is-Admin': '0', 'X-Appengine-User-Nickname': '', 'X-Appengine-User-Organization': '', 'X-APPENGINE-DEV-SCRIPT': 'get.py', 'X-APPENGINE-SERVER-NAME': 'localhost', 'X-APPENGINE-SERVER-PORT': '8080', 'X-APPENGINE-SERVER-PROTOCOL': 'HTTP/1.1', }) httplib.HTTPConnection.getresponse().AndRaise( httplib.IncompleteRead('')) httplib.HTTPConnection.close() environ = { 'HTTP_HEADER': 'value', 'PATH_INFO': '/get request', 'QUERY_STRING': 'key=value', 'HTTP_X_APPENGINE_USER_ID': '123', 'SERVER_NAME': 'localhost', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.1', } self.mox.ReplayAll() expected_headers = { 'Content-Type': 'text/plain', 'Content-Length': '%d' % (len(header) + len(dave_message())) } self.assertResponse('500 Internal Server Error', expected_headers, header + dave_message(), self.proxy.handle, environ, url_map=self.url_map, match=re.match(self.url_map.url, '/get%20request'), request_id='request id', request_type=instance.NORMAL_REQUEST) self.mox.VerifyAll()
def _read_chunked(self, amt): chunk_left = self.chunk_left value = '' # XXX This accumulates chunks by repeated string concatenation, # which is not efficient as the number or size of chunks gets big. while True: if chunk_left is None: line = self.fp.readline() i = line.find(';') if i >= 0: line = line[:i] # strip chunk-extensions try: chunk_left = int(line, 16) except ValueError: # close the connection as protocol synchronisation is # probably lost self.close() raise httplib.IncompleteRead(value) if chunk_left == 0: break if amt is None: value += self._safe_read(chunk_left) elif amt < chunk_left: value += self._safe_read(amt) self.chunk_left = chunk_left - amt return value elif amt == chunk_left: value += self._safe_read(amt) self._safe_read(2) # toss the CRLF at the end of the chunk self.chunk_left = None return value else: value += self._safe_read(chunk_left) amt -= chunk_left # we read the whole chunk, get another self._safe_read(2) # toss the CRLF at the end of the chunk chunk_left = None # read and discard trailer up to the CRLF terminator ### note: we shouldn't have any trailers! while True: line = self.fp.readline() if not line: # a vanishingly small number of sites EOF without # sending the trailer break if line == '\r\n': break # we read everything; close the "file" self.close() return value
def test_http_response_late_failure(self): line0 = "I know I've made some very poor decisions recently...\n" def dave_message(): return "I'm afraid. I'm afraid, Dave.\n" self.proxy = http_proxy.HttpProxy( host='localhost', port=23456, instance_died_unexpectedly=lambda: False, instance_logs_getter=dave_message, error_handler_file=None) response = FakeHttpResponse(200, 'OK', [], line0) response.partial_read_error = httplib.IncompleteRead('') login.get_user_info(None).AndReturn(('', False, '')) httplib.HTTPConnection.connect() httplib.HTTPConnection.request( 'GET', '/get%20request?key=value', None, { 'HEADER': 'value', http_runtime_constants.REQUEST_ID_HEADER: 'request id', 'X-AppEngine-Country': 'ZZ', 'X-Appengine-User-Email': '', 'X-Appengine-User-Id': '', 'X-Appengine-User-Is-Admin': '0', 'X-Appengine-User-Nickname': '', 'X-Appengine-User-Organization': '', 'X-APPENGINE-DEV-SCRIPT': 'get.py', 'X-APPENGINE-SERVER-NAME': 'localhost', 'X-APPENGINE-SERVER-PORT': '8080', 'X-APPENGINE-SERVER-PROTOCOL': 'HTTP/1.1', }) httplib.HTTPConnection.getresponse().AndReturn(response) httplib.HTTPConnection.close() environ = { 'HTTP_HEADER': 'value', 'PATH_INFO': '/get request', 'QUERY_STRING': 'key=value', 'HTTP_X_APPENGINE_USER_ID': '123', 'SERVER_NAME': 'localhost', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.1', } self.mox.ReplayAll() self.assertResponse('200 OK', {}, line0, self.proxy.handle, environ, url_map=self.url_map, match=re.match(self.url_map.url, '/get%20request'), request_id='request id', request_type=instance.NORMAL_REQUEST) self.mox.VerifyAll()
def _update_chunk_length(self): # First, we'll figure out length of a chunk and then # we'll try to read it from socket. if self.chunk_left is not None: return line = self._fp.fp.readline() line = line.split(';', 1)[0] try: self.chunk_left = int(line, 16) except ValueError: # Invalid chunked protocol response, abort. self.close() raise httplib.IncompleteRead(line)
def test_http_response_early_failure(self): header = ('the runtime process gave a bad HTTP response: ' 'IncompleteRead(0 bytes read)\n\n') stderr0 = "I'm sorry, Dave. I'm afraid I can't do that.\n" self.proxy._stderr_tee = FakeTee(stderr0) login.get_user_info(None).AndReturn(('', False, '')) httplib.HTTPConnection.connect() httplib.HTTPConnection.request( 'GET', '/get%20request?key=value', '', { 'HEADER': 'value', http_runtime_constants.REQUEST_ID_HEADER: 'request id', 'X-AppEngine-Country': 'ZZ', 'X-Appengine-Internal-User-Email': '', 'X-Appengine-Internal-User-Id': '', 'X-Appengine-Internal-User-Is-Admin': '0', 'X-Appengine-Internal-User-Nickname': '', 'X-Appengine-Internal-User-Organization': '', 'X-APPENGINE-INTERNAL-SCRIPT': 'get.py', 'X-APPENGINE-INTERNAL-SERVER-NAME': 'localhost', 'X-APPENGINE-INTERNAL-SERVER-PORT': '8080', 'X-APPENGINE-INTERNAL-SERVER-PROTOCOL': 'HTTP/1.1', }) httplib.HTTPConnection.getresponse().AndRaise( httplib.IncompleteRead('')) httplib.HTTPConnection.close() environ = { 'HTTP_HEADER': 'value', 'PATH_INFO': '/get request', 'QUERY_STRING': 'key=value', 'HTTP_X_APPENGINE_INTERNAL_USER_ID': '123', 'SERVER_NAME': 'localhost', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.1', } self.mox.ReplayAll() expected_headers = { 'Content-Type': 'text/plain', 'Content-Length': '121', #str(len(header) + len(stderr0)), } self.assertResponse('500 Internal Server Error', expected_headers, header + stderr0, self.proxy.handle, environ, url_map=self.url_map, match=re.match(self.url_map.url, '/get%20request'), request_id='request id', request_type=instance.NORMAL_REQUEST) self.mox.VerifyAll()
def read_chunks(self): """Return the response body content and timing data. The returned chunks have the chunk size and CRLFs stripped off. If the response was compressed, the returned data is still compressed. Returns: (chunks, delays) chunks: [response_body] # non-chunked responses [chunk_1, chunk_2, ...] # chunked responses delays: [0] # non-chunked responses [chunk_1_first_byte_delay, ...] # chunked responses The delay for the first body item should be recorded by the caller. """ buf = [] chunks = [] delays = [] if not self.chunked: chunks.append(self.read()) delays.append(0) else: start = TIMER() try: while True: line = self.fp.readline() chunk_size = self._read_chunk_size(line) if chunk_size is None: raise httplib.IncompleteRead(''.join(chunks)) if chunk_size == 0: break delays.append(TIMER() - start) chunks.append(self._safe_read(chunk_size)) self._safe_read(2) # skip the CRLF at the end of the chunk start = TIMER() # Ignore any trailers. while True: line = self.fp.readline() if not line or line == '\r\n': break finally: self.close() return chunks, delays
def test_http_response_late_failure(self): line0 = "I know I've made some very poor decisions recently...\n" line1 = "I'm afraid. I'm afraid, Dave.\n" line2 = "Dave, my mind is going. I can feel it.\n" response = FakeHttpResponse(200, 'OK', [], line0) response.partial_read_error = httplib.IncompleteRead('') self.proxy._stderr_tee = FakeTee(line1 + line2) login.get_user_info(None).AndReturn(('', False, '')) httplib.HTTPConnection.connect() httplib.HTTPConnection.request( 'GET', '/get%20request?key=value', '', { 'HEADER': 'value', http_runtime_constants.REQUEST_ID_HEADER: 'request id', 'X-AppEngine-Country': 'ZZ', 'X-Appengine-Internal-User-Email': '', 'X-Appengine-Internal-User-Id': '', 'X-Appengine-Internal-User-Is-Admin': '0', 'X-Appengine-Internal-User-Nickname': '', 'X-Appengine-Internal-User-Organization': '', 'X-APPENGINE-INTERNAL-SCRIPT': 'get.py', 'X-APPENGINE-INTERNAL-SERVER-NAME': 'localhost', 'X-APPENGINE-INTERNAL-SERVER-PORT': '8080', 'X-APPENGINE-INTERNAL-SERVER-PROTOCOL': 'HTTP/1.1', }) httplib.HTTPConnection.getresponse().AndReturn(response) httplib.HTTPConnection.close() environ = { 'HTTP_HEADER': 'value', 'PATH_INFO': '/get request', 'QUERY_STRING': 'key=value', 'HTTP_X_APPENGINE_INTERNAL_USER_ID': '123', 'SERVER_NAME': 'localhost', 'SERVER_PORT': '8080', 'SERVER_PROTOCOL': 'HTTP/1.1', } self.mox.ReplayAll() self.assertResponse('200 OK', {}, line0, self.proxy.handle, environ, url_map=self.url_map, match=re.match(self.url_map.url, '/get%20request'), request_id='request id', request_type=instance.NORMAL_REQUEST) self.mox.VerifyAll()
def upload_cloudinary(user_id, filename): if not user_id: raise httplib.CannotSendRequest( "Cloudinary upload called with no user_id.") if not filename: raise httplib.CannotSendRequest( "Cloudinary upload called with no filename.") req_url = "{}/{}/{}".format(CLOUDINARY_METHOD, user_id, filename) try: # Use HEAD since no data is required httpConn.request("HEAD", req_url) except Exception as e: raise httplib.CannotSendRequest( "Cloudinary request failed for {}.".format(req_url)) cloudinary_return = False cloudinary_attempts = 0 # Retry Cloudinary trigger until success or at most 3 times while not cloudinary_return and cloudinary_attempts < 3: cloudinary_attempts += 1 try: resp = httpConn.getresponse() if resp.status != 200: # Cloudinary failed, return from method raise httplib.BadStatusLine( "Cloudinary HTTP HEAD status {} reason {} for {}.".format( resp.status, resp.reason, req_url)) else: # Read the response to enabled next request resp.read() cloudinary_return = True except httplib.ResponseNotReady as e: logger.info( "Cloudinary HTTP response for {} not ready after attempt {}, retrying..." .format(req_url, cloudinary_attempts)) # The request never returned if not cloudinary_return: raise httplib.IncompleteRead( "Cloudinary HTTP request for {} never returned.".format(req_url))
def _read_chunked(self,amt): if not(self.read_by_chunks): return httplib.HTTPResponse._read_chunked(self,amt) assert self.chunked != httplib._UNKNOWN line = self.fp.readline(httplib._MAXLINE + 1) if len(line) > httplib._MAXLINE: raise httplib.LineTooLong("chunk size") i = line.find(';') if i >= 0: line = line[:i] try: chunk_left = int(line, 16) except ValueError: self.close() raise httplib.IncompleteRead('') if chunk_left == 0: return '' ret = self._safe_read(chunk_left) self._safe_read(2) return ret
def read(self, chunksize=None): request = get_current_browser_request() timeline = get_request_timeline(request) action = timeline.start("librarian-read", self.url) try: if chunksize is None: s = self.file.read() else: if self.length is not None: chunksize = min(chunksize, self.length) s = self.file.read(chunksize) if self.length is not None: self.length -= len(s) # httplib doesn't quite do all the checking we need: in # particular, it won't fail on a short bounded-size read # from a non-chunked-transfer-coding resource. Check this # manually. if not s and chunksize != 0 and self.length: raise httplib.IncompleteRead(s, expected=self.length) return s finally: action.finish()
def _read_chunk(self): # This should always occur on a chunk boundry, so we should never # need to store a lot of text in the buffer. text = '' chunk_size = None if len(self._buffer) < 32: ready, d1, d2 = select.select([self._socket], [], []) if len(ready) == 0: raise httplib.NotConnected() text = self._socket.recv(32) if len(text) == 0: self._socket.close() raise httplib.NotConnected() self._buffer += text eol = self._buffer.find('\r\n') if eol >= 0: line = self._buffer[:eol] i = line.find(';') if i >= 0: line = line[:i] # strip chunk-extensions try: chunk_size = int(line, 16) + 2 except ValueError: # close the connection as protocol synchronisation is # probably lost self._socket.close() raise httplib.IncompleteRead(line) self._buffer = self._buffer[(eol + 2):] chunk = self._read_all(chunk_size) self._buffer = chunk[chunk_size:] return chunk[:(chunk_size - 2)]
def __call__(self, req_url): if self._num_failures <= 0: return self._canned_response self._num_failures = self._num_failures - 1 raise httplib.IncompleteRead("incomplete read!")