def _gae_urlfetch(appid, payload, getfast, method, realurl):
request_params, http_util, connection_cache_key = _get_request_params(appid)
if http_util is http_gws:
request_headers = {
'User-Agent': 'Mozilla/5.0',
'Accept-Encoding': 'gzip',
'Content-Length': str(len(payload))
}
else:
#禁用 CDN 不兼容的 GAE chunked 机制
request_headers = {
'User-Agent': '',
'Content-Length': str(len(payload))
}
while True:
response = http_util.request(request_params, payload, request_headers,
connection_cache_key=connection_cache_key,
getfast=getfast,
realmethod=method,
realurl=realurl)
if response is None:
return
if response.status not in (200, 404):
break
if http_util is http_nor:
break
app_server = response.headers.get('Server')
if app_server == 'Google Frontend':
break
if GC.GAE_ENABLEPROXY:
logging.warning('GAE 前置代理 [%s:%d] 无法正常工作', *response.xip)
continue
if test_ip_gae(response.xip[0]):
break
logging.warning('发现并移除非 GAE IP：%s，Server：%s', response.xip[0], app_server)
response.http_util = http_util
response.connection_cache_key = connection_cache_key
response.app_status = response.status
if response.status != 200:
return response
#解压并解析 chunked & gziped 响应
if 'Transfer-Encoding' in response.headers:
responseg = HTTPResponse(GzipSock(response), method=method)
responseg.begin()
responseg.app_status = 200
responseg.xip = response.xip
responseg.sock = response.sock
responseg.http_util = http_util
responseg.connection_cache_key = connection_cache_key
return responseg
#读取压缩头部
data = response.read(2)
if len(data) < 2:
response.status = 502
make_errinfo(response, 'connection aborted. too short leadtype data=%r' % data)
return response
headers_length, = struct.unpack('!h', data)
data = response.read(headers_length)
if len(data) < headers_length:
response.status = 502
make_errinfo(response, 'connection aborted. too short headers data=%r' % data)
return response
#解压缩并解析头部
raw_response_line, headers_data = zlib.decompress(data, -zlib.MAX_WBITS).split(b'\r\n', 1)
raw_response_line = str(raw_response_line, 'iso-8859-1')
raw_response_list = raw_response_line.split(None, 2)
raw_response_length = len(raw_response_list)
if raw_response_length == 3:
_, status, reason = raw_response_list
response.reason = reason.strip()
elif raw_response_length == 2:
_, status = raw_response_list
response.reason = ''
else:
return
response.status = int(status)
#标记服务器端错误信息
headers_data, app_msg = headers_data.split(b'\r\n\r\n')
if app_msg:
response.app_status = response.status
response.reason = 'debug error'
response.app_msg = app_msg
response.headers = response.msg = parse_headers(BytesIO(headers_data))
if response.app_status == 200:
response._method = method
if response.status in (204, 205, 304) or 100 <= response.status < 200:
response.length = 0
else:
try:
response.length = int(response.headers.get('Content-Length'))
except:
response.length = None
return response
