def request(self, method, path, body="", headers=None):
"""
Requests made via this connection actually get translated and routed
into our WSGI app, we then wait for the response and turn it back into
an `httplib.HTTPResponse`.
"""
if not headers:
headers = {}
req = webob.Request.blank(path)
req.method = method
req.headers = headers
req.host = self.host
req.body = body
resp = str(req.get_response(self.app))
resp = "HTTP/1.0 %s" % resp
sock = FakeHttplibSocket(resp)
self.http_response = httplib.HTTPResponse(sock)
self.http_response.begin()
def make_response(self, sock, req):
""" 根据socket和urlib2.Request 构建Response """
sock.setblocking(True)
data = req.get_data()
method = "POST" if data else "GET"
r = httplib.HTTPResponse(sock,
0,
strict=0,
method=method,
buffering=True)
r.begin()
r.recv = r.read
fp = socket._fileobject(r, close=True)
resp = urllib.addinfourl(fp, r.msg, req.get_full_url())
resp.code = r.status
resp.msg = r.reason
self.cookiejar.extract_cookies(resp, req)
self.cookiejar.save()
return resp
def test_app_check(ssl_sock, ip):
request_data = 'GET /check HTTP/1.1\r\nHost: xxnet-check.appspot.com\r\n\r\n'
time_start = time.time()
ssl_sock.send(request_data.encode())
response = httplib.HTTPResponse(ssl_sock, buffering=True)
try:
response.begin()
status = response.status
if status != 200:
xlog.debug("app check %s status:%d", ip, status)
raise Exception("app check fail")
content = response.read()
if not content == "CHECK_OK":
xlog.debug("app check %s content:%s", ip, content)
raise Exception("content fail")
finally:
response.close()
time_stop = time.time()
time_cost = (time_stop - time_start) * 1000
xlog.debug("app check time:%d", time_cost)
return True
def test_incomplete_read(self):
if test_support.due_to_ironpython_incompatibility(
"http://www.codeplex.com/IronPython/WorkItem/View.aspx?WorkItemId=21116"
):
return
sock = FakeSocket(
'HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\nHello\r\n')
resp = httplib.HTTPResponse(sock, method="GET")
resp.begin()
try:
resp.read()
except httplib.IncompleteRead as i:
self.assertEquals(i.partial, 'Hello\r\n')
self.assertEqual(repr(i),
"IncompleteRead(7 bytes read, 3 more expected)")
self.assertEqual(str(i),
"IncompleteRead(7 bytes read, 3 more expected)")
else:
self.fail('IncompleteRead expected')
finally:
resp.close()
def fetch(self, method, host, path, headers, payload, bufsize=8192, timeout=20):
request_data = '%s %s HTTP/1.1\r\n' % (method, path)
request_data += ''.join('%s: %s\r\n' % (k, v) for k, v in headers.items())
request_data += '\r\n'
#print("request:%s" % request_data)
#print("payload:%s" % payload)
conn = self.get_conn()
if not conn:
logging.warn("get sock fail")
return
if len(request_data) + len(payload) < 1300:
payload = request_data.encode() + payload
else:
conn.sock.send(request_data.encode())
payload_len = len(payload)
start = 0
while start < payload_len:
send_size = min(payload_len - start, 65535)
sended = conn.sock.send(payload[start:start+send_size])
start += sended
conn.sock.settimeout(timeout)
response = httplib.HTTPResponse(conn.sock, buffering=True)
response.conn = conn
try:
#orig_timeout = conn.sock.gettimeout()
#conn.sock.settimeout(timeout)
response.begin()
#conn.sock.settimeout(orig_timeout)
except httplib.BadStatusLine as e:
logging.warn("fetch bad status line:%r", e)
response = None
except Exception as e:
logging.warn("fetch:%r", e)
return response
def SOCKET_SEND(http_packet):
# SEND REQUEST
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
ssl_s = ssl.wrap_socket(s)
ssl_s.connect((host_address, https_port))
if SHOW_HTTP_REQUESTS:
print("--- Sending ---\r\n {} \r\n----".format(http_packet))
if PYTHON == 2:
ssl_s.send(http_packet)
else:
ssl_s.send(bytes(http_packet, 'UTF-8'))
# GET RESPONSE
response = ssl_s.recv(1024)
ssl_s.close()
if SHOW_HTTP_REQUESTS:
print("--- Response --- \r\n {} \r\n---")
# PARSE REPONSE
fake_socket_response = FakeSocket(response)
parsed_response = httplib.HTTPResponse(fake_socket_response)
parsed_response.begin()
return parsed_response
def test_app_head(ssl_sock, ip):
appid = appid_manager.get_appid()
request_data = 'HEAD /_gh/ HTTP/1.1\r\nHost: %s.appspot.com\r\n\r\n' % appid
time_start = time.time()
ssl_sock.send(request_data.encode())
response = httplib.HTTPResponse(ssl_sock, buffering=True)
try:
response.begin()
status = response.status
if status != 200:
xlog.debug("app check %s status:%d", ip, status)
raise Exception("app check fail")
except Exception as e:
xlog.exception("test_app_head except:%r", e)
return False
finally:
response.close()
time_stop = time.time()
time_cost = (time_stop - time_start) * 1000
xlog.debug("app check time:%d", time_cost)
return True
def __init__(self, response):
r = httplib.HTTPResponse(self._FakeSocket(response))
r.begin()
self.location = r.getheader("location", '')
self.usn = r.getheader("usn", '')
self.st = r.getheader("st", '')
try:
self.cache = r.getheader("cache-control").split("=")[1]
except:
logging.warning("Could not extract cache-control header.")
logging.warning("Headers are: %s", r.getheaders())
index = self.usn.find('::')
if index >= 0:
self.uuid = self.usn[5:index]
else:
self.uuid = self.usn[5:]
self.type = SSDPResponse.ST_UNKNOWN
if self.st == 'upnp:rootdevice':
self.type = SSDPResponse.ST_ROOT_DEVICE
elif self.st.startswith('urn:schemas-upnp-org:device'):
self.type = SSDPResponse.ST_DEVICE
self.deviceType = self.st[28:]
def test_server_type(ssl_sock, ip):
request_data = "HEAD / HTTP/1.1\r\nAccept: */*\r\nHost: %s\r\n\r\n" % ip
time_start = time.time()
ssl_sock.send(request_data.encode())
response = httplib.HTTPResponse(ssl_sock, buffering=True)
try:
response.begin()
server_type = response.msg.dict["server"]
time_stop = time.time()
time_cost = (time_stop - time_start) * 1000
server_type = server_type.replace(" ", "_") # gvs 1.0
if server_type == 'HTTP_server_(unknown)':
res_url = response.msg.dict["location"]
if "google.com/sorry/IndexRedirect?" in res_url:
scan_sleep()
if server_type == '': # for avoid csv split
server_type = '_'
xlog.info("server_type:%s time:%d", server_type, time_cost)
return server_type
finally:
response.close()
def _request(sock, headers, payload, bufsize=8192):
request_data = 'POST /_gh/ HTTP/1.1\r\n'
request_data += ''.join('%s: %s\r\n' % (k, v) for k, v in headers.items()
if k not in skip_headers)
request_data += '\r\n'
if isinstance(payload, bytes):
sock.send(request_data.encode())
payload_len = len(payload)
start = 0
while start < payload_len:
send_size = min(payload_len - start, 65535)
sended = sock.send(payload[start:start + send_size])
start += sended
elif hasattr(payload, 'read'):
sock.send(request_data)
while True:
data = payload.read(bufsize)
if not data:
break
sock.send(data)
else:
raise TypeError(
'_request(payload) must be a string or buffer, not %r' %
type(payload))
response = httplib.HTTPResponse(sock, buffering=True)
try:
orig_timeout = sock.gettimeout()
sock.settimeout(100)
response.begin()
sock.settimeout(orig_timeout)
except httplib.BadStatusLine as e:
#logging.warn("_request bad status line:%r", e)
response.close()
response = None
except Exception as e:
xlog.warn("_request:%r", e)
return response
def check_goagent(ssl_sock, appid):
request_data = 'GET /_gh/ HTTP/1.1\r\nHost: %s.appspot.com\r\n\r\n' % appid
ssl_sock.send(request_data.encode())
response = httplib.HTTPResponse(ssl_sock, buffering=True)
response.begin()
if response.status == 404:
if __name__ == "__main__":
xlog.warn("app check %s status:%d", appid, response.status)
return False
if response.status == 503:
# out of quota
server_type = response.getheader('Server', "")
if "gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type:
if __name__ == "__main__":
xlog.warn("503 but server type:%s", server_type)
return False
else:
if __name__ == "__main__":
xlog.info("503 server type:%s", server_type)
return True
if response.status != 200:
if __name__ == "__main__":
xlog.warn("app check %s ip:%s status:%d", appid, ip,
response.status)
return False
content = response.read()
if "GoAgent" not in content:
if __name__ == "__main__":
xlog.warn("app check %s content:%s", appid, content)
return False
if __name__ == "__main__":
xlog.info("check_goagent ok")
return True
def handle_read(self):
if self.can_receive:
try:
# When response is received, read the headers
# and the status
if not self.response:
# Need to set blocking for the response to be
# able to parse headers
self.socket.setblocking(1)
self.response = httplib.HTTPResponse(self.socket)
self.response.begin()
self.response.async_handler = self
self.header_callback(self)
# Continue reading the body in case the callback doesnt expect
# a raw response
if self.can_receive:
self.socket.setblocking(0)
asynchat.async_chat.handle_read(self)
except ssl.SSLError:
print "Got ssl error "
pass
def test_appid_exist(ssl_sock, appid):
request_data = 'GET /_gh/ HTTP/1.1\r\nHost: %s.appspot.com\r\n\r\n' % appid
ssl_sock.send(request_data.encode())
response = httplib.HTTPResponse(ssl_sock, buffering=True)
response.begin()
if response.status == 404:
# xlog.warn("app check %s status:%d", appid, response.status)
return False
if response.status == 503:
# out of quota
return True
if response.status != 200:
xlog.warn("test appid %s status:%d", appid, response.status)
content = response.read()
if "GoAgent" not in content:
# xlog.warn("app check %s content:%s", appid, content)
return False
return True
def proxy_ssl(self):
host = '%s:%d' % (self.host, self.port)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((self.proxy, int(self.proxy_port)))
except:
raise
sock.sendall("CONNECT %s HTTP/1.0\r\n" % host)
sock.sendall("User-Agent: %s\r\n" % UserAgent)
if self.proxy_user and self.proxy_pass:
for k, v in self.get_proxy_auth_header().items():
sock.sendall("%s: %s\r\n" % (k, v))
sock.sendall("\r\n")
resp = httplib.HTTPResponse(sock, strict=True)
resp.begin()
if resp.status != 200:
# Fake a socket error, use a code that make it obvious it hasn't
# been generated by the socket library
raise socket.error(
-71, "Error talking to HTTP proxy %s:%s: %s (%s)" %
(self.proxy, self.proxy_port, resp.status, resp.reason))
# We can safely close the response, it duped the original socket
resp.close()
h = httplib.HTTPConnection(host)
# Wrap the socket in an SSL socket
if hasattr(httplib, 'ssl'):
sslSock = httplib.ssl.SSLSocket(sock)
else: # Old Python, no ssl module
sslSock = socket.ssl(sock, None, None)
sslSock = httplib.FakeSocket(sock, sslSock)
# This is a bit unclean
h.sock = sslSock
return h
def queryRequest(self,
method="GET",
uri="/",
version="HTTP/1.0",
numHeaders=1):
try:
sock = self._connect(self.host, self.port)
startline = self.createStartLine(method, uri, version)
testheader = "x-summitsage-test-header: this is a test header\r\n"
self.send(sock, startline)
self.send(sock, "\r\n")
print startline + " " + str(numHeaders)
while numHeaders > 0:
self.send(sock, testheader)
numHeaders = numHeaders - 1
self.send(sock, "\r\n")
except socket.error:
print "socket error sending queryRequest"
return
try:
response = httplib.HTTPResponse(sock, 0, True)
response.begin()
response.read()
print "Status: " + str(response.status)
except socket.error:
print "socket error reading queryRequest"
except httplib.BadStatusLine:
print "response had bad status line."
except IOError:
print "an IOError occured"
sock.close()
def _getresponse(self):
''' Read from recv and return a HTTPResponse object if possible.
Either
1 - The client has succesfully closed the connection: Return ''
2 - The server has already closed the connection: Return the response
if possible.
'''
# Wait for a response
self._conn.sock.setblocking(True)
# Parse the response
response = self._bytes
while True:
try:
bytes = self._conn.sock.recv(1)
except httplib.socket.error:
# For error 54: Connection reset by peer
# (and perhaps others)
return ''
if bytes == '':
break
else:
response += bytes
# Set recv to be non-blocking again
self._conn.sock.setblocking(False)
# Convert the response string to a httplib.HTTPResponse
# object with a bit of a hack
if response != '':
# Taken from
# http://pythonwise.blogspot.ca/2010/02/parse-http-response.html
try:
response = httplib.HTTPResponse(_FakeSocket(response))
response.begin()
except:
# Bad headers ... etc.
response = ''
return response
def _log_response(self, r):
rv = "# -------- begin %d:%d response ----------\n" % (id(self), id(r))
ht = ""
v = r.version
if r.version == 10:
v = "HTTP/1.0"
if r.version == 11:
v = "HTTP/1.1"
ht += "%s %s %s\r\n" % (v, r.status, r.reason)
body = r.read()
for h in r.getheaders():
ht += "%s: %s\r\n" % (h[0].title(), h[1])
ht += "\r\n"
# this is evil. laugh with me. ha arharhrhahahaha
class fakesock:
def __init__(self, s):
self.s = s
def makefile(self, mode, foo):
return StringIO.StringIO(self.s)
rr = r
if r.chunked:
ht += "%x\r\n" % (len(body))
ht += body
ht += "\r\n0\r\n"
else:
ht += body
rr = httplib.HTTPResponse(fakesock(ht),
method=r._method,
debuglevel=r.debuglevel)
rr.begin()
rv += ht
rv += ("\n# -------- end %d:%d response ----------\n" %
(id(self), id(r)))
return (rr, rv)
def head_request(self):
# for keep alive
# public appid don't keep alive, for quota limit.
if self.ssl_sock.appid in config.PUBLIC_APPIDS:
#xlog.info("public appid don't keep alive")
return False
start_time = time.time()
# xlog.debug("head request %s", host)
request_data = 'HEAD /_gh/ HTTP/1.1\r\nHost: %s\r\n\r\n' % self.ssl_sock.host
try:
data = request_data.encode()
ret = self.ssl_sock.send(data)
if ret != len(data):
xlog.warn("head send len:%d %d", ret, len(data))
response = httplib.HTTPResponse(self.ssl_sock, buffering=True)
self.ssl_sock.settimeout(100)
response.begin()
status = response.status
if status != 200:
xlog.debug("app head fail status:%d", status)
raise Exception("app check fail %r" % status)
self.rtt = (time.time() - start_time) * 1000
return True
except httplib.BadStatusLine as e:
time_now = time.time()
inactive_time = time_now - self.ssl_sock.last_use_time
head_timeout = time_now - start_time
xlog.warn("%s keep alive fail, inactive_time:%d head_timeout:%d",
self.ssl_sock.ip, inactive_time, head_timeout)
except Exception as e:
xlog.exception("%s head %s request fail:%r", self.ssl_sock.ip,
self.ssl_sock.appid, e)
def head_request(ssl_sock):
if ssl_sock.host == '':
ssl_sock.appid = appid_manager.get_appid()
if not ssl_sock.appid:
logging.error("no appid can use")
return False
host = ssl_sock.appid + ".appspot.com"
ssl_sock.host = host
else:
host = ssl_sock.host
#logging.debug("head request %s", host)
request_data = 'HEAD /_gh/ HTTP/1.1\r\nHost: %s\r\n\r\n' % host
try:
ssl_sock.settimeout(2)
ssl_sock.send(request_data.encode())
response = httplib.HTTPResponse(ssl_sock, buffering=True)
response.begin()
status = response.status
if status != 200:
logging.debug("app head fail status:%d", status)
raise Exception("app check fail")
content = response.read()
return True
except httplib.BadStatusLine as e:
return False
except Exception as e:
logging.debug("head request fail:%r", e)
return False
finally:
response.close()
def contentLengthBodyTest(self,
body="some data",
method="GET",
uri="/",
version="HTTP/1.0"):
try:
sock = self._connect(self.host, self.port)
startline = self.createStartLine(method, uri, version)
self.send(sock, startline)
self.send(sock, "\r\n")
contentLength = "content-length: " + str(len(body)) + "\r\n\r\n"
print contentLength,
self.send(sock, contentLength)
for c in body:
sock.send(c)
time.sleep(1)
sys.stdout.write(c)
sys.stdout.flush()
except socket.error:
print "socket error occured during content length test."
try:
response = httplib.HTTPResponse(sock, 0, True)
response.begin()
response.read()
print "Status: " + str(response.status)
except socket.error:
print "socket error reading content length test"
except httplib.BadStatusLine:
print "content length test response had bad status line."
except IOError:
print "an IOError occured"
def test_parse_all_octets(self):
# Ensure no valid header field octet breaks the parser
body = (
b'HTTP/1.1 200 OK\r\n'
b"!#$%&'*+-.^_`|~: value\r\n" # Special token characters
b'VCHAR: ' + bytearray(range(0x21, 0x7E + 1)) + b'\r\n'
b'obs-text: ' + bytearray(range(0x80, 0xFF + 1)) + b'\r\n'
b'obs-fold: text\r\n'
b' folded with space\r\n'
b'\tfolded with tab\r\n'
b'Content-Length: 0\r\n'
b'\r\n')
sock = FakeSocket(body)
resp = httplib.HTTPResponse(sock)
resp.begin()
self.assertEqual(resp.getheader('Content-Length'), '0')
self.assertEqual(resp.getheader("!#$%&'*+-.^_`|~"), 'value')
vchar = ''.join(map(chr, range(0x21, 0x7E + 1)))
self.assertEqual(resp.getheader('VCHAR'), vchar)
self.assertIsNotNone(resp.getheader('obs-text'))
folded = resp.getheader('obs-fold')
self.assertTrue(folded.startswith('text'))
self.assertIn(' folded with space', folded)
self.assertTrue(folded.endswith('folded with tab'))
def parse_headers(all_headers):
# all_headers contains headers from all responses - even without FOLLOWLOCATION there
# might be multiple sets of headers due to 401 Unauthorized. We only care about the last
# response.
try:
raw_headers = all_headers.split(b"\r\n\r\n")[-2]
except IndexError:
logger.warning('Incorrectly terminated http headers')
raw_headers = all_headers
logger.debug("raw headers: " + repr(raw_headers))
# http://stackoverflow.com/questions/24728088/python-parse-http-response-string/24729316#24729316
class FakeSocket(object):
def __init__(self, response_str):
self._file = BytesIO(response_str)
def makefile(self, *args, **kwargs):
return self._file
response = httplib.HTTPResponse(FakeSocket(raw_headers))
response.begin()
header_list = [(k.lower(), v) for (k, v) in response.getheaders()]
return dict(header_list)
def head_request(self):
# for keep alive, not work now.
start_time = time.time()
# xlog.debug("head request %s", self.ssl_sock.ip)
request_data = 'GET / HTTP/1.1\r\nHost: %s\r\n\r\n' % self.ssl_sock.host
try:
data = request_data.encode()
ret = self.ssl_sock.send(data)
if ret != len(data):
xlog.warn("h1 head send len:%r %d %s", ret, len(data), self.ip)
return False
response = httplib.HTTPResponse(self.ssl_sock, buffering=True)
self.ssl_sock.settimeout(100)
response.begin()
status = response.status
if status != 200:
xlog.debug("%s appid:%s head fail status:%d", self.ip,
self.ssl_sock.appid, status)
return False
content = response.read()
self.rtt = (time.time() - start_time) * 1000
return True
except httplib.BadStatusLine as e:
time_now = time.time()
inactive_time = time_now - self.last_active_time
head_timeout = time_now - start_time
xlog.debug("%s keep alive fail, inactive_time:%d head_timeout:%d",
self.ssl_sock.ip, inactive_time, head_timeout)
except Exception as e:
xlog.debug("h1 %s appid:%s HEAD keep alive request fail:%r",
self.ssl_sock.ip, self.ssl_sock.appid, e)
def _request(self,
sock,
method,
path,
protocol_version,
headers,
payload,
bufsize=8192):
skip_headers = self.skip_headers
request_data = '%s %s %s\r\n' % (method, path, protocol_version)
request_data += ''.join('%s: %s\r\n' % (k, v)
for k, v in headers.items()
if k not in skip_headers)
request_data += '\r\n'
if isinstance(payload, bytes):
sock.sendall(request_data.encode() + payload)
elif hasattr(payload, 'read'):
sock.sendall(request_data)
while 1:
data = payload.read(bufsize)
if not data:
break
sock.sendall(data)
else:
raise TypeError(
'http_util.request(payload) must be a string or buffer, not %r'
% type(payload))
response = httplib.HTTPResponse(sock, buffering=True)
try:
response.begin()
except httplib.BadStatusLine:
response = None
return response
def _test():
# Test HTTP status lines
body = "HTTP/1.1 200 Ok\r\n\r\nText"
sock = FakeSocket(body)
resp = httplib.HTTPResponse(sock, 1)
resp.begin()
print resp.read()
resp.close()
body = "HTTP/1.1 400.100 Not Ok\r\n\r\nText"
sock = FakeSocket(body)
resp = httplib.HTTPResponse(sock, 1)
try:
resp.begin()
except httplib.BadStatusLine:
print "BadStatusLine raised as expected"
else:
print "Expect BadStatusLine"
# Check invalid host_port
for hp in ("www.python.org:abc", "www.python.org:"):
try:
h = httplib.HTTP(hp)
except httplib.InvalidURL:
print "InvalidURL raised as expected"
else:
print "Expect InvalidURL"
for hp,h,p in (("[fe80::207:e9ff:fe9b]:8000", "fe80::207:e9ff:fe9b", 8000),
("www.python.org:80", "www.python.org", 80),
("www.python.org", "www.python.org", 80),
("[fe80::207:e9ff:fe9b]", "fe80::207:e9ff:fe9b", 80)):
try:
http = httplib.HTTP(hp)
except httplib.InvalidURL:
print "InvalidURL raised erroneously"
c = http._conn
if h != c.host: raise AssertionError, ("Host incorrectly parsed", h, c.host)
if p != c.port: raise AssertionError, ("Port incorrectly parsed", p, c.host)
# test response with multiple message headers with the same field name.
text = ('HTTP/1.1 200 OK\r\n'
'Set-Cookie: Customer="WILE_E_COYOTE"; Version="1"; Path="/acme"\r\n'
'Set-Cookie: Part_Number="Rocket_Launcher_0001"; Version="1";'
' Path="/acme"\r\n'
'\r\n'
'No body\r\n')
hdr = ('Customer="WILE_E_COYOTE"; Version="1"; Path="/acme"'
', '
'Part_Number="Rocket_Launcher_0001"; Version="1"; Path="/acme"')
s = FakeSocket(text)
r = httplib.HTTPResponse(s, 1)
r.begin()
cookies = r.getheader("Set-Cookie")
if cookies != hdr:
raise AssertionError, "multiple headers not combined properly"
# Test that the library doesn't attempt to read any data
# from a HEAD request. (Tickles SF bug #622042.)
sock = FakeSocket(
'HTTP/1.1 200 OK\r\n'
'Content-Length: 14432\r\n'
'\r\n',
NoEOFStringIO)
resp = httplib.HTTPResponse(sock, 1, method="HEAD")
resp.begin()
if resp.read() != "":
raise AssertionError, "Did not expect response from HEAD request"
resp.close()
def raw_to_httpresponse(s):
""" translate a raw tornado http response into an httplib.HTTPResponse """
sock = FakeHttplibSocket(s)
resp = httplib.HTTPResponse(sock)
resp.begin()
return resp
def test_overflowing_status_line(self):
self.skipTest("disabled for HTTP 0.9 support")
body = "HTTP/1.1 200 Ok" + "k" * 65536 + "\r\n"
resp = httplib.HTTPResponse(FakeSocket(body))
self.assertRaises((httplib.LineTooLong, httplib.BadStatusLine), resp.begin)
def test_too_many_headers(self):
headers = '\r\n'.join('Header%d: foo' % i for i in xrange(200)) + '\r\n'
text = ('HTTP/1.1 200 OK\r\n' + headers)
s = FakeSocket(text)
r = httplib.HTTPResponse(s)
self.assertRaises(httplib.HTTPException, r.begin)
def test_overflowing_header_line(self):
body = ('HTTP/1.1 200 OK\r\n' 'X-Foo: bar' + 'r' * 65536 + '\r\n\r\n')
resp = httplib.HTTPResponse(FakeSocket(body))
self.assertRaises(httplib.LineTooLong, resp.begin)
def _request(self, headers, payload):
request_data = 'POST /_gh/ HTTP/1.1\r\n'
request_data += ''.join('%s: %s\r\n' % (k, v)
for k, v in headers.items())
request_data += '\r\n'
try:
self.ssl_sock.send(request_data.encode())
payload_len = len(payload)
start = 0
while start < payload_len:
send_size = min(payload_len - start, 65535)
sended = self.ssl_sock.send(payload[start:start + send_size])
start += sended
response = httplib.HTTPResponse(self.ssl_sock, buffering=True)
self.ssl_sock.settimeout(100)
response.begin()
# read response body,
body_length = int(response.getheader("Content-Length", "0"))
start = 0
end = body_length - 1
last_read_time = time.time()
time_response = time.time()
response_body = []
while True:
if start > end:
self.ssl_sock.received_size += body_length
response.headers = response.msg.dict
response.body = ReadBuffer(b''.join(response_body))
response.ssl_sock = self.ssl_sock
response.worker = self
return response
data = response.read(65535)
if not data:
if time.time() - last_read_time > 20:
google_ip.report_connect_closed(
self.ssl_sock.ip, "down fail")
response.close()
xlog.warn("%s read timeout t:%d len:%d left:%d ",
self.ip,
(time.time() - time_response) * 1000,
body_length, (end - start))
return False
else:
time.sleep(0.1)
continue
last_read_time = time.time()
data_len = len(data)
start += data_len
response_body.append(data)
except httplib.BadStatusLine as e:
xlog.warn("%s _request bad status line:%r", self.ip, e)
pass
except Exception as e:
xlog.warn("%s _request:%r", self.ip, e)
return False
