def gae_urlfetch(method, url, headers, payload, appid, getfast=None, **kwargs): # GAE 代理请求不允许设置 Host 头域 if 'Host' in headers: del headers['Host'] metadata = '%s %s HTTP/1.1\r\n' % (method, url) metadata += ''.join('%s: %s\r\n' % (k, v) for k, v in headers.items()) metadata += gae_options if not isinstance(metadata, bytes): metadata = metadata.encode() metadata = zlib.compress(metadata)[2:-4] if payload: if not isinstance(payload, bytes): payload = payload.encode() payload = struct.pack('!h', len(metadata)) + metadata + payload else: payload = struct.pack('!h', len(metadata)) + metadata request_headers = { 'User-Agent': 'Mozilla/5.0', 'Accept-Encoding': 'gzip', 'Content-Length': str(len(payload)) } request_params = gae_params_dict[appid] realurl = 'GAE-' + url qGAE.get() # get start from Queue while True: response = http_gws.request(request_params, payload, request_headers, connection_cache_key='google_gae|:443', getfast=getfast, realmethod=method, realurl=realurl) if response is None: return if response.status not in (200, 404): break app_server = response.headers.get('Server') if app_server == 'Google Frontend': break if GC.GAE_ENABLEPROXY: logging.warning('GAE 前置代理 [%s:%d] 无法正常工作', *response.xip) continue if test_ip_gae(response.xip[0]): break logging.warning('发现并移除非 GAE IP:%s,Server:%s', response.xip[0], app_server) response.app_status = response.status if response.status != 200: return response #解压并解析 chunked & gziped 响应 if 'Transfer-Encoding' in response.headers: responseg = HTTPResponse(GzipSock(response), method=method) responseg.begin() responseg.app_status = 200 responseg.xip = response.xip responseg.sock = response.sock return responseg #读取压缩头部 data = response.read(2) if len(data) < 2: response.status = 502 make_errinfo(response, b'connection aborted. too short leadtype data=' + data) return response headers_length, = struct.unpack('!h', data) data = response.read(headers_length) if len(data) < headers_length: response.status = 502 make_errinfo(response, b'connection aborted. too short headers data=' + data) return response #解压缩并解析头部 raw_response_line, headers_data = zlib.decompress(data, -zlib.MAX_WBITS).split(b'\r\n', 1) raw_response_line = str(raw_response_line, 'iso-8859-1') raw_response_list = raw_response_line.split(None, 2) raw_response_length = len(raw_response_list) if raw_response_length == 3: _, status, reason = raw_response_list response.reason = reason.strip() elif raw_response_length == 2: _, status = raw_response_list response.reason = '' else: return response.status = int(status) #标记服务器端错误信息 headers_data, app_msg = headers_data.split(b'\r\n\r\n') if app_msg: response.app_status = response.status response.reason = 'debug error' response.app_msg = app_msg response.headers = response.msg = parse_headers(BytesIO(headers_data)) if response.app_status == 200: response._method = method if response.status in (204, 205, 304) or 100 <= response.status < 200: response.length = 0 else: try: response.length = int(response.headers.get('Content-Length')) except: response.length = None return response
def _gae_urlfetch(appid, payload, getfast, method, realurl): request_params, http_util, connection_cache_key = _get_request_params(appid) if http_util is http_gws: request_headers = { 'User-Agent': 'Mozilla/5.0', 'Accept-Encoding': 'gzip', 'Content-Length': str(len(payload)) } else: #禁用 CDN 不兼容的 GAE chunked 机制 request_headers = { 'User-Agent': '', 'Content-Length': str(len(payload)) } while True: response = http_util.request(request_params, payload, request_headers, connection_cache_key=connection_cache_key, getfast=getfast, realmethod=method, realurl=realurl) if response is None: return if response.status not in (200, 404): break if http_util is http_nor: break app_server = response.headers.get('Server') if app_server == 'Google Frontend': break if GC.GAE_ENABLEPROXY: logging.warning('GAE 前置代理 [%s:%d] 无法正常工作', *response.xip) continue if test_ip_gae(response.xip[0]): break logging.warning('发现并移除非 GAE IP:%s,Server:%s', response.xip[0], app_server) response.http_util = http_util response.connection_cache_key = connection_cache_key response.app_status = response.status if response.status != 200: return response #解压并解析 chunked & gziped 响应 if 'Transfer-Encoding' in response.headers: responseg = HTTPResponse(GzipSock(response), method=method) responseg.begin() responseg.app_status = 200 responseg.xip = response.xip responseg.sock = response.sock responseg.http_util = http_util responseg.connection_cache_key = connection_cache_key return responseg #读取压缩头部 data = response.read(2) if len(data) < 2: response.status = 502 make_errinfo(response, 'connection aborted. too short leadtype data=%r' % data) return response headers_length, = struct.unpack('!h', data) data = response.read(headers_length) if len(data) < headers_length: response.status = 502 make_errinfo(response, 'connection aborted. too short headers data=%r' % data) return response #解压缩并解析头部 raw_response_line, headers_data = zlib.decompress(data, -zlib.MAX_WBITS).split(b'\r\n', 1) raw_response_line = str(raw_response_line, 'iso-8859-1') raw_response_list = raw_response_line.split(None, 2) raw_response_length = len(raw_response_list) if raw_response_length == 3: _, status, reason = raw_response_list response.reason = reason.strip() elif raw_response_length == 2: _, status = raw_response_list response.reason = '' else: return response.status = int(status) #标记服务器端错误信息 headers_data, app_msg = headers_data.split(b'\r\n\r\n') if app_msg: response.app_status = response.status response.reason = 'debug error' response.app_msg = app_msg response.headers = response.msg = parse_headers(BytesIO(headers_data)) if response.app_status == 200: response._method = method if response.status in (204, 205, 304) or 100 <= response.status < 200: response.length = 0 else: try: response.length = int(response.headers.get('Content-Length')) except: response.length = None return response