def request_gae_server(headers, body): # process on http protocol # process status code return by http server # raise error, let up layer retry. response = http_dispatch.request(headers, body) if not response: raise GAE_Exception(600, "fetch gae fail") if response.status >= 600: raise GAE_Exception(response.status, "fetch gae fail:%d" % response.status) server_type = response.headers.get("server", "") content_type = response.headers.get("content-type", "") if ("gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type) or \ response.status == 403 or response.status == 405: # some ip can connect, and server type can be gws # but can't use as GAE server # so we need remove it immediately xlog.warn("IP:%s not support GAE, server:%s status:%d", response.ssl_sock.ip, server_type, response.status) google_ip.recheck_ip(response.ssl_sock.ip) response.worker.close("ip not support GAE") raise GAE_Exception(602, "ip not support GAE") if response.status == 404: # xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid) appid_manager.report_not_exist(response.ssl_sock.appid, response.ssl_sock.ip) # google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist") response.worker.close("appid not exist:%s" % response.ssl_sock.appid) raise GAE_Exception(603, "appid not support GAE") if response.status == 503: appid = response.ssl_sock.appid xlog.warning('APPID %r out of Quota, remove it. %s', appid, response.ssl_sock.ip) appid_manager.report_out_of_quota(appid) # google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota") response.worker.close("appid out of quota:%s" % appid) raise GAE_Exception(604, "appid out of quota:%s" % appid) if response.status > 300: raise GAE_Exception(605, "status:%d" % response.status) if response.status != 200: xlog.warn("GAE %s appid:%s status:%d", response.ssl_sock.ip, response.ssl_sock.appid, response.status) else: xlog.info('good ip num:%d, bad ip num:%s', google_ip.good_ip_num, google_ip.bad_ip_num) return response
def request_gae_server(headers, body, url, timeout): # process on http protocol # process status code return by http server # raise error, let up layer retry. response = http_dispatch.request(headers, body, url, timeout) if not response: raise GAE_Exception(600, "fetch gae fail") if response.status >= 600: raise GAE_Exception( response.status, "fetch gae fail:%d" % response.status) server_type = response.getheader("server", "") # content_type = response.getheaders("content-type", "") if ("gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type) or \ response.status == 403 or response.status == 405: # some ip can connect, and server type can be gws # but can't use as GAE server # so we need remove it immediately xlog.warn("IP:%s not support GAE, headers:%s status:%d", response.ssl_sock.ip, response.headers, response.status) google_ip.recheck_ip(response.ssl_sock.ip) response.worker.close("ip not support GAE") raise GAE_Exception(602, "ip not support GAE") if response.status == 404: # xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid) appid_manager.report_not_exist( response.ssl_sock.appid, response.ssl_sock.ip) # google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist") response.worker.close("appid not exist:%s" % response.ssl_sock.appid) raise GAE_Exception(603, "appid not support GAE") if response.status == 503: appid = response.ssl_sock.appid xlog.warning('APPID %r out of Quota, remove it. %s', appid, response.ssl_sock.ip) appid_manager.report_out_of_quota(appid) # google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota") response.worker.close("appid out of quota:%s" % appid) raise GAE_Exception(604, "appid out of quota:%s" % appid) if response.status > 300: raise GAE_Exception(605, "status:%d" % response.status) if response.status != 200: xlog.warn("GAE %s appid:%s status:%d", response.ssl_sock.ip, response.ssl_sock.appid, response.status) return response
def request_gae_server(headers, body, url, timeout): response = http_dispatch.request(headers, body, url, timeout) if not response: raise GAE_Exception(600, "fetch gae fail") if response.status >= 600: raise GAE_Exception(response.status, "fetch gae fail:%d" % response.status) server_type = response.getheader("server", "") if ("gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type) or \ response.status == 403 or response.status == 405: xlog.warn("IP:%s not support GAE, headers:%s status:%d", response.ssl_sock.ip, response.headers, response.status) google_ip.recheck_ip(response.ssl_sock.ip) response.worker.close("ip not support GAE") raise GAE_Exception(602, "ip not support GAE") if response.status == 404: # xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid) appid_manager.report_not_exist(response.ssl_sock.appid, response.ssl_sock.ip) # google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist") response.worker.close("appid not exist:%s" % response.ssl_sock.appid) raise GAE_Exception(603, "appid not support GAE") if response.status == 503: appid = response.ssl_sock.appid xlog.warning('APPID %r out of Quota, remove it. %s', appid, response.ssl_sock.ip) appid_manager.report_out_of_quota(appid) # google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota") response.worker.close("appid out of quota:%s" % appid) raise GAE_Exception(604, "appid out of quota:%s" % appid) if response.status > 300: raise GAE_Exception(605, "status:%d" % response.status) if response.status != 200: xlog.warn("GAE %s appid:%s status:%d", response.ssl_sock.ip, response.ssl_sock.appid, response.status) else: xlog.info('good ip num:%d, bad ip num:%s', google_ip.good_ip_num, google_ip.bad_ip_num) return response
def request_gae_server(headers, body, url, timeout): response = http_dispatch.request(headers, body, url, timeout) if not response: raise GAE_Exception(600, "fetch gae fail") if response.status >= 600: raise GAE_Exception( response.status, "fetch gae fail:%d" % response.status) server_type = response.getheader("server", "") if ("gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type) or \ response.status == 403 or response.status == 405: xlog.warn("IP:%s not support GAE, headers:%s status:%d", response.ssl_sock.ip, response.headers, response.status) google_ip.recheck_ip(response.ssl_sock.ip) response.worker.close("ip not support GAE") raise GAE_Exception(602, "ip not support GAE") if response.status == 404: # xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid) appid_manager.report_not_exist( response.ssl_sock.appid, response.ssl_sock.ip) # google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist") response.worker.close("appid not exist:%s" % response.ssl_sock.appid) raise GAE_Exception(603, "appid not support GAE") if response.status == 503: appid = response.ssl_sock.appid xlog.warning('APPID %r out of Quota, remove it. %s', appid, response.ssl_sock.ip) appid_manager.report_out_of_quota(appid) # google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota") response.worker.close("appid out of quota:%s" % appid) raise GAE_Exception(604, "appid out of quota:%s" % appid) if response.status > 300: raise GAE_Exception(605, "status:%d" % response.status) if response.status != 200: xlog.warn("GAE %s appid:%s status:%d", response.ssl_sock.ip, response.ssl_sock.appid, response.status) else: xlog.info('good ip num:%d, bad ip num:%s', google_ip.good_ip_num, google_ip.bad_ip_num) return response
def fetch_by_gae(method, url, headers, body): if isinstance(body, basestring) and body: if len(body) < 10 * 1024 * 1024 and 'Content-Encoding' not in headers: zbody = deflate(body) if len(zbody) < len(body): body = zbody headers['Content-Encoding'] = 'deflate' if len(body) > 10 * 1024 * 1024: xlog.warn("body len:%d %s %s", len(body), method, url) headers['Content-Length'] = str(len(body)) # GAE don't allow set `Host` header if 'Host' in headers: del headers['Host'] kwargs = {} if config.GAE_PASSWORD: kwargs['password'] = config.GAE_PASSWORD #kwargs['options'] = #kwargs['validate'] = kwargs['maxsize'] = config.AUTORANGE_MAXSIZE kwargs['timeout'] = '19' payload = '%s %s HTTP/1.1\r\n' % (method, url) payload += ''.join('%s: %s\r\n' % (k, v) for k, v in headers.items() if k not in skip_headers) #for k, v in headers.items(): # xlog.debug("Send %s: %s", k, v) payload += ''.join('X-URLFETCH-%s: %s\r\n' % (k, v) for k, v in kwargs.items() if v) request_headers = {} payload = deflate(payload) body = '%s%s%s' % (struct.pack('!h', len(payload)), payload, body) request_headers['Content-Length'] = str(len(body)) response = http_dispatch.request(request_headers, body) response.app_status = response.status response.app_headers = response.headers if response.app_status != 200: return response check_local_network.report_network_ok() try: data = response.body.get(2) if len(data) < 2: xlog.warn("fetch too short lead byte len:%d %s", len(data), url) response.app_status = 502 # 502: Bad gateway response.fp = io.BytesIO( b'connection aborted. too short lead byte data=' + data) response.read = response.fp.read return response headers_length, = struct.unpack('!h', data) data = response.body.get(headers_length) if len(data) < headers_length: xlog.warn("fetch too short header need:%d get:%d %s", headers_length, len(data), url) response.app_status = 509 response.fp = io.BytesIO( b'connection aborted. too short headers data=' + data) response.read = response.fp.read return response raw_response_line, headers_data = inflate(data).split('\r\n', 1) _, response.status, response.reason = raw_response_line.split(None, 2) response.status = int(response.status) response.reason = response.reason.strip() headers_pairs = headers_data.split('\r\n') response.headers = {} for pair in headers_pairs: if not pair: break k, v = pair.split(': ', 1) response.headers[k] = v return response except Exception as e: raise GAE_Exception("unpack protocol:%r", e)
def fetch_by_gae(method, url, headers, body): if isinstance(body, basestring) and body: if len(body) < 10 * 1024 * 1024 and 'Content-Encoding' not in headers: zbody = deflate(body) if len(zbody) < len(body): body = zbody headers['Content-Encoding'] = 'deflate' if len(body) > 10 * 1024 * 1024: xlog.warn("body len:%d %s %s", len(body), method, url) headers['Content-Length'] = str(len(body)) # GAE don't allow set `Host` header if 'Host' in headers: del headers['Host'] kwargs = {} if config.GAE_PASSWORD: kwargs['password'] = config.GAE_PASSWORD #kwargs['options'] = #kwargs['validate'] = kwargs['maxsize'] = config.AUTORANGE_MAXSIZE kwargs['timeout'] = '19' payload = '%s %s HTTP/1.1\r\n' % (method, url) payload += ''.join('%s: %s\r\n' % (k, v) for k, v in headers.items() if k not in skip_headers) #for k, v in headers.items(): # xlog.debug("Send %s: %s", k, v) payload += ''.join('X-URLFETCH-%s: %s\r\n' % (k, v) for k, v in kwargs.items() if v) request_headers = {} payload = deflate(payload) body = '%s%s%s' % (struct.pack('!h', len(payload)), payload, body) request_headers['Content-Length'] = str(len(body)) response = http_dispatch.request(request_headers, body) response.app_status = response.status response.app_headers = response.headers if response.app_status != 200: return response check_local_network.report_network_ok() try: data = response.body.get(2) if len(data) < 2: xlog.warn("fetch too short lead byte len:%d %s", len(data), url) response.app_status = 502 # 502: Bad gateway response.fp = io.BytesIO(b'connection aborted. too short lead byte data=' + data) response.read = response.fp.read return response headers_length, = struct.unpack('!h', data) data = response.body.get(headers_length) if len(data) < headers_length: xlog.warn("fetch too short header need:%d get:%d %s", headers_length, len(data), url) response.app_status = 509 response.fp = io.BytesIO(b'connection aborted. too short headers data=' + data) response.read = response.fp.read return response raw_response_line, headers_data = inflate(data).split('\r\n', 1) _, response.status, response.reason = raw_response_line.split(None, 2) response.status = int(response.status) response.reason = response.reason.strip() headers_pairs = headers_data.split('\r\n') response.headers = {} for pair in headers_pairs: if not pair: break k, v = pair.split(': ', 1) response.headers[k] = v return response except Exception as e: raise GAE_Exception("unpack protocol:%r", e)