def request_gae_proxy(method, url, headers, body): time_request = time.time() while True: if time.time() - time_request > 60: #time out return False try: response = fetch_by_gae(method, url, headers, body) if response.app_status < 300: return response xlog.warn("fetch gae status:%s url:%s", response.app_status, url) if response.app_status == 506: # fetch fail at http request continue server_type = response.app_headers.get('Server', "") if "gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type: xlog.warn("IP:%s not support GAE, server type:%s", response.ssl_sock.ip, server_type) google_ip.report_connect_fail(response.ssl_sock.ip, force_remove=True) response.worker.close("ip not support GAE") continue if response.app_status == 404: # xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid) appid_manager.report_not_exist(response.ssl_sock.appid, response.ssl_sock.ip) # google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist") response.worker.close("appid not exist:%s" % response.ssl_sock.appid) continue if response.app_status == 403 or response.app_status == 405: #Method not allowed # google have changed from gws to gvs, need to remove. xlog.warning('405 Method not allowed. remove %s ', response.ssl_sock.ip) # some ip can connect, and server type is gws # but can't use as GAE server # so we need remove it immediately google_ip.report_connect_fail(response.ssl_sock.ip, force_remove=True) response.worker.close("ip not support GAE") continue if response.app_status == 503: xlog.warning('APPID %r out of Quota, remove it. %s', response.ssl_sock.appid, response.ssl_sock.ip) appid_manager.report_out_of_quota(response.ssl_sock.appid) # google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota") response.worker.close("appid out of quota") continue except GAE_Exception as e: xlog.warn("gae_exception:%r %s", e, url) except Exception as e: xlog.exception('gae_handler.handler %r %s , retry...', e, url)
def request_gae_server(headers, body): # process on http protocol # process status code return by http server # raise error, let up layer retry. response = http_dispatch.request(headers, body) if not response: raise GAE_Exception(600, "fetch gae fail") if response.status >= 600: raise GAE_Exception(response.status, "fetch gae fail:%d" % response.status) server_type = response.headers.get("server", "") content_type = response.headers.get("content-type", "") if ("gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type) or \ response.status == 403 or response.status == 405: # some ip can connect, and server type can be gws # but can't use as GAE server # so we need remove it immediately xlog.warn("IP:%s not support GAE, server:%s status:%d", response.ssl_sock.ip, server_type, response.status) google_ip.recheck_ip(response.ssl_sock.ip) response.worker.close("ip not support GAE") raise GAE_Exception(602, "ip not support GAE") if response.status == 404: # xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid) appid_manager.report_not_exist(response.ssl_sock.appid, response.ssl_sock.ip) # google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist") response.worker.close("appid not exist:%s" % response.ssl_sock.appid) raise GAE_Exception(603, "appid not support GAE") if response.status == 503: appid = response.ssl_sock.appid xlog.warning('APPID %r out of Quota, remove it. %s', appid, response.ssl_sock.ip) appid_manager.report_out_of_quota(appid) # google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota") response.worker.close("appid out of quota:%s" % appid) raise GAE_Exception(604, "appid out of quota:%s" % appid) if response.status > 300: raise GAE_Exception(605, "status:%d" % response.status) if response.status != 200: xlog.warn("GAE %s appid:%s status:%d", response.ssl_sock.ip, response.ssl_sock.appid, response.status) else: xlog.info('good ip num:%d, bad ip num:%s', google_ip.good_ip_num, google_ip.bad_ip_num) return response
def request_gae_server(headers, body, url, timeout): # process on http protocol # process status code return by http server # raise error, let up layer retry. response = http_dispatch.request(headers, body, url, timeout) if not response: raise GAE_Exception(600, "fetch gae fail") if response.status >= 600: raise GAE_Exception( response.status, "fetch gae fail:%d" % response.status) server_type = response.getheader("server", "") # content_type = response.getheaders("content-type", "") if ("gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type) or \ response.status == 403 or response.status == 405: # some ip can connect, and server type can be gws # but can't use as GAE server # so we need remove it immediately xlog.warn("IP:%s not support GAE, headers:%s status:%d", response.ssl_sock.ip, response.headers, response.status) google_ip.recheck_ip(response.ssl_sock.ip) response.worker.close("ip not support GAE") raise GAE_Exception(602, "ip not support GAE") if response.status == 404: # xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid) appid_manager.report_not_exist( response.ssl_sock.appid, response.ssl_sock.ip) # google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist") response.worker.close("appid not exist:%s" % response.ssl_sock.appid) raise GAE_Exception(603, "appid not support GAE") if response.status == 503: appid = response.ssl_sock.appid xlog.warning('APPID %r out of Quota, remove it. %s', appid, response.ssl_sock.ip) appid_manager.report_out_of_quota(appid) # google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota") response.worker.close("appid out of quota:%s" % appid) raise GAE_Exception(604, "appid out of quota:%s" % appid) if response.status > 300: raise GAE_Exception(605, "status:%d" % response.status) if response.status != 200: xlog.warn("GAE %s appid:%s status:%d", response.ssl_sock.ip, response.ssl_sock.appid, response.status) return response
def request_gae_server(headers, body, url, timeout): response = http_dispatch.request(headers, body, url, timeout) if not response: raise GAE_Exception(600, "fetch gae fail") if response.status >= 600: raise GAE_Exception(response.status, "fetch gae fail:%d" % response.status) server_type = response.getheader("server", "") if ("gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type) or \ response.status == 403 or response.status == 405: xlog.warn("IP:%s not support GAE, headers:%s status:%d", response.ssl_sock.ip, response.headers, response.status) google_ip.recheck_ip(response.ssl_sock.ip) response.worker.close("ip not support GAE") raise GAE_Exception(602, "ip not support GAE") if response.status == 404: # xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid) appid_manager.report_not_exist(response.ssl_sock.appid, response.ssl_sock.ip) # google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist") response.worker.close("appid not exist:%s" % response.ssl_sock.appid) raise GAE_Exception(603, "appid not support GAE") if response.status == 503: appid = response.ssl_sock.appid xlog.warning('APPID %r out of Quota, remove it. %s', appid, response.ssl_sock.ip) appid_manager.report_out_of_quota(appid) # google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota") response.worker.close("appid out of quota:%s" % appid) raise GAE_Exception(604, "appid out of quota:%s" % appid) if response.status > 300: raise GAE_Exception(605, "status:%d" % response.status) if response.status != 200: xlog.warn("GAE %s appid:%s status:%d", response.ssl_sock.ip, response.ssl_sock.appid, response.status) else: xlog.info('good ip num:%d, bad ip num:%s', google_ip.good_ip_num, google_ip.bad_ip_num) return response
def request_gae_proxy(method, url, headers, body, timeout=60, retry=True): # make retry and time out time_request = time.time() request_headers, request_body = pack_request(method, url, headers, body) error_msg = [] while True: if time.time() - time_request > timeout: raise GAE_Exception(600, b"".join(error_msg)) if not retry and error_msg: raise GAE_Exception(600, b"".join(error_msg)) try: response = request_gae_server(request_headers, request_body, url, timeout) check_local_network.report_network_ok() response = unpack_response(response) if response.app_msg: xlog.warn("server app return fail, status:%d", response.app_status) # if len(response.app_msg) < 2048: #xlog.warn('app_msg:%s', cgi.escape(response.app_msg)) if response.app_status == 510: # reach 80% of traffic today # disable for get big file. appid_manager.report_out_of_quota(response.ssl_sock.appid) response.worker.close( "appid out of quota:%s" % response.ssl_sock.appid) continue return response except GAE_Exception as e: err_msg = "gae_exception:%r %s" % (e, url) error_msg.append(err_msg) xlog.warn("gae_exception:%r %s", e, url) except Exception as e: err_msg = 'gae_handler.handler %r %s , retry...' % (e, url) error_msg.append(err_msg) xlog.exception('gae_handler.handler %r %s , retry...', e, url)
def request_gae_server(headers, body, url, timeout): response = http_dispatch.request(headers, body, url, timeout) if not response: raise GAE_Exception(600, "fetch gae fail") if response.status >= 600: raise GAE_Exception( response.status, "fetch gae fail:%d" % response.status) server_type = response.getheader("server", "") if ("gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type) or \ response.status == 403 or response.status == 405: xlog.warn("IP:%s not support GAE, headers:%s status:%d", response.ssl_sock.ip, response.headers, response.status) google_ip.recheck_ip(response.ssl_sock.ip) response.worker.close("ip not support GAE") raise GAE_Exception(602, "ip not support GAE") if response.status == 404: # xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid) appid_manager.report_not_exist( response.ssl_sock.appid, response.ssl_sock.ip) # google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist") response.worker.close("appid not exist:%s" % response.ssl_sock.appid) raise GAE_Exception(603, "appid not support GAE") if response.status == 503: appid = response.ssl_sock.appid xlog.warning('APPID %r out of Quota, remove it. %s', appid, response.ssl_sock.ip) appid_manager.report_out_of_quota(appid) # google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota") response.worker.close("appid out of quota:%s" % appid) raise GAE_Exception(604, "appid out of quota:%s" % appid) if response.status > 300: raise GAE_Exception(605, "status:%d" % response.status) if response.status != 200: xlog.warn("GAE %s appid:%s status:%d", response.ssl_sock.ip, response.ssl_sock.appid, response.status) else: xlog.info('good ip num:%d, bad ip num:%s', google_ip.good_ip_num, google_ip.bad_ip_num) return response
def request_gae_proxy(method, url, headers, body, timeout=60, retry=True): # make retry and time out time_request = time.time() request_headers, request_body = pack_request(method, url, headers, body) error_msg = [] while True: if time.time() - time_request > timeout: raise GAE_Exception(600, b"".join(error_msg)) if not retry and error_msg: raise GAE_Exception(600, b"".join(error_msg)) try: response = request_gae_server(request_headers, request_body, url, timeout) response = unpack_response(response) if response.app_msg: xlog.warn("server app return fail, status:%d", response.app_status) # if len(response.app_msg) < 2048: #xlog.warn('app_msg:%s', cgi.escape(response.app_msg)) if response.app_status == 510: # reach 80% of traffic today # disable for get big file. appid_manager.report_out_of_quota(response.ssl_sock.appid) response.worker.close( "appid out of quota:%s" % response.ssl_sock.appid) continue return response except GAE_Exception as e: err_msg = "gae_exception:%r %s" % (e, url) error_msg.append(err_msg) xlog.warn("gae_exception:%r %s", e, url) except Exception as e: err_msg = 'gae_handler.handler %r %s , retry...' % (e, url) error_msg.append(err_msg) xlog.exception('gae_handler.handler %r %s , retry...', e, url)
def __fetchlet(self, range_queue, data_queue, range_delay_size): headers = dict((k.title(), v) for k, v in self.headers.items()) headers['Connection'] = 'close' while not self._stopped: try: try: start, end, response = range_queue.get(timeout=1) if self.expect_begin < start and data_queue.qsize() * self.bufsize + range_delay_size > 30*1024*1024: range_queue.put((start, end, response)) time.sleep(10) continue headers['Range'] = 'bytes=%d-%d' % (start, end) if not response: response = fetch(self.method, self.url, headers, self.body) except Queue.Empty: continue except Exception as e: xlog.warning("RangeFetch fetch response %r in __fetchlet", e) range_queue.put((start, end, None)) continue if not response: xlog.warning('RangeFetch %s return %r', headers['Range'], response) range_queue.put((start, end, None)) continue if response.app_status != 200: xlog.warning('Range Fetch return %s "%s %s" %s ', response.app_status, self.method, self.url, headers['Range']) if response.app_status == 404: xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid) appid_manager.report_not_exist(response.ssl_sock.appid, response.ssl_sock.ip) appid = appid_manager.get_appid() if not appid: xlog.error("no appid left") self._stopped = True response.close() return if response.app_status == 503: xlog.warning('APPID %r out of Quota, remove it temporary.', response.ssl_sock.appid) appid_manager.report_out_of_quota(response.ssl_sock.appid) appid = appid_manager.get_appid() if not appid: xlog.error("no appid left") self._stopped = True response.close() return google_ip.report_connect_closed(response.ssl_sock.ip, "app err") response.close() range_queue.put((start, end, None)) continue if response.getheader('Location'): self.url = urlparse.urljoin(self.url, response.getheader('Location')) xlog.info('RangeFetch Redirect(%r)', self.url) google_ip.report_connect_closed(response.ssl_sock.ip, "reLocation") response.close() range_queue.put((start, end, None)) continue if 200 <= response.status < 300: content_range = response.getheader('Content-Range') if not content_range: xlog.warning('RangeFetch "%s %s" return Content-Range=%r: response headers=%r, retry %s-%s', self.method, self.url, content_range, response.getheaders(), start, end) google_ip.report_connect_closed(response.ssl_sock.ip, "no range") response.close() range_queue.put((start, end, None)) continue content_length = int(response.getheader('Content-Length', 0)) xlog.info('>>>>>>>>>>>>>>> [thread %s] %s %s', threading.currentThread().ident, content_length, content_range) time_last_read = time.time() while start < end + 1: try: data = response.read(self.bufsize) if not data: if time.time() - time_last_read > 20: break else: time.sleep(0.1) continue time_last_read = time.time() data_len = len(data) data_queue.put((start, data)) start += data_len except Exception as e: xlog.warning('RangeFetch "%s %s" %s failed: %s', self.method, self.url, headers['Range'], e) break if start < end + 1: xlog.warning('RangeFetch "%s %s" retry %s-%s', self.method, self.url, start, end) google_ip.report_connect_closed(response.ssl_sock.ip, "down err") response.close() range_queue.put((start, end, None)) continue https_manager.save_ssl_connection_for_reuse(response.ssl_sock) xlog.info('>>>>>>>>>>>>>>> Successfully reached %d bytes.', start - 1) else: xlog.error('RangeFetch %r return %s', self.url, response.status) google_ip.report_connect_closed(response.ssl_sock.ip, "status err") response.close() range_queue.put((start, end, None)) continue except StandardError as e: xlog.exception('RangeFetch._fetchlet error:%s', e) raise
def handler(method, url, headers, body, wfile): time_request = time.time() headers = clean_empty_header(headers) errors = [] response = None while True: if time.time() - time_request > 30: #time out return return_fail_message(wfile) try: response = fetch(method, url, headers, body) if response.app_status != 200: xlog.warn("fetch gae status:%s url:%s", response.app_status, url) try: server_type = response.getheader('Server', "") if "gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type: xlog.warn("IP:%s not support GAE, server type:%s", response.ssl_sock.ip, server_type) google_ip.report_connect_fail(response.ssl_sock.ip, force_remove=True) response.close() continue except Exception as e: errors.append(e) xlog.warn('gae_handler.handler %r %s , retry...', e, url) continue if response.app_status == 404: #xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid) appid_manager.report_not_exist(response.ssl_sock.appid, response.ssl_sock.ip) google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist") appid = appid_manager.get_appid() if not appid: html = generate_message_html('404 No usable Appid Exists', u'没有可用appid了,请配置可用的appid') send_response(wfile, 404, body=html.encode('utf-8')) response.close() return else: response.close() continue if response.app_status == 403 or response.app_status == 405: #Method not allowed # google have changed from gws to gvs, need to remove. xlog.warning('405 Method not allowed. remove %s ', response.ssl_sock.ip) # some ip can connect, and server type is gws # but can't use as GAE server # so we need remove it immediately google_ip.report_connect_fail(response.ssl_sock.ip, force_remove=True) response.close() continue if response.app_status == 503: xlog.warning('APPID %r out of Quota, remove it. %s', response.ssl_sock.appid, response.ssl_sock.ip) appid_manager.report_out_of_quota(response.ssl_sock.appid) google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota") appid = appid_manager.get_appid() if not appid: html = generate_message_html('503 No usable Appid Exists', u'appid流量不足,请增加appid') send_response(wfile, 503, body=html.encode('utf-8')) response.close() return else: response.close() continue if response.app_status < 500: break except GAE_Exception as e: errors.append(e) xlog.warn("gae_exception:%r %s", e, url) except Exception as e: errors.append(e) xlog.exception('gae_handler.handler %r %s , retry...', e, url) if response.status == 206: return RangeFetch(method, url, headers, body, response, wfile).fetch() try: wfile.write("HTTP/1.1 %d %s\r\n" % (response.status, response.reason)) response_headers = {} for key, value in response.getheaders(): key = key.title() if key == 'Transfer-Encoding': #http://en.wikipedia.org/wiki/Chunked_transfer_encoding continue if key in skip_headers: continue response_headers[key] = value if 'X-Head-Content-Length' in response_headers: if method == "HEAD": response_headers['Content-Length'] = response_headers['X-Head-Content-Length'] del response_headers['X-Head-Content-Length'] send_to_browser = True try: for key in response_headers: value = response_headers[key] send_header(wfile, key, value) #logging.debug("Head- %s: %s", key, value) wfile.write("\r\n") except Exception as e: send_to_browser = False xlog.warn("gae_handler.handler send response fail. t:%d e:%r %s", time.time()-time_request, e, url) if len(response.app_msg): xlog.warn("APPID error:%d url:%s", response.status, url) wfile.write(response.app_msg) google_ip.report_connect_closed(response.ssl_sock.ip, "app err") response.close() return content_length = int(response.getheader('Content-Length', 0)) content_range = response.getheader('Content-Range', '') if content_range: start, end, length = tuple(int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)', content_range).group(1, 2, 3)) else: start, end, length = 0, content_length-1, content_length body_length = end - start + 1 last_read_time = time.time() time_response = time.time() while True: if start > end: time_finished = time.time() if body_length > 1024 and time_finished - time_response > 0: speed = body_length / (time_finished - time_response) xlog.info("GAE %d|%s|%d t:%d s:%d hs:%d Spd:%d %d %s", response.ssl_sock.fd, response.ssl_sock.ip, response.ssl_sock.received_size, (time_finished-time_request)*1000, length, response.ssl_sock.handshake_time, int(speed), response.status, url) else: xlog.info("GAE %d|%s|%d t:%d s:%d hs:%d %d %s", response.ssl_sock.fd, response.ssl_sock.ip, response.ssl_sock.received_size, (time_finished-time_request)*1000, length, response.ssl_sock.handshake_time, response.status, url) response.ssl_sock.received_size += body_length https_manager.save_ssl_connection_for_reuse(response.ssl_sock, call_time=time_request) return data = response.read(config.AUTORANGE_BUFSIZE) if not data: if time.time() - last_read_time > 20: google_ip.report_connect_closed(response.ssl_sock.ip, "down fail") response.close() xlog.warn("read timeout t:%d len:%d left:%d %s", (time.time()-time_request)*1000, length, (end-start), url) return else: time.sleep(0.1) continue last_read_time = time.time() data_len = len(data) start += data_len if send_to_browser: try: ret = wfile.write(data) if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ: xlog.debug("send to browser wfile.write ret:%d", ret) ret = wfile.write(data) except Exception as e_b: if e_b[0] in (errno.ECONNABORTED, errno.EPIPE, errno.ECONNRESET) or 'bad write retry' in repr(e_b): xlog.warn('gae_handler send to browser return %r %r', e_b, url) else: xlog.warn('gae_handler send to browser return %r %r', e_b, url) send_to_browser = False except NetWorkIOError as e: time_except = time.time() time_cost = time_except - time_request if e[0] in (errno.ECONNABORTED, errno.EPIPE) or 'bad write retry' in repr(e): xlog.warn("gae_handler err:%r time:%d %s ", e, time_cost, url) google_ip.report_connect_closed(response.ssl_sock.ip, "Net") else: xlog.exception("gae_handler except:%r %s", e, url) except Exception as e: xlog.exception("gae_handler except:%r %s", e, url)
def handler(method, url, headers, body, wfile): time_request = time.time() errors = [] response = None for i in xrange(max_retry): try: response = fetch(method, url, headers, body) if response.app_status != 200: logging.debug("fetch gae status:%d url:%s", response.app_status, url) if response.app_status == 404: logging.warning('APPID %r not exists, remove it.', response.ssl_sock.appid) appid_manager.report_not_exist(response.ssl_sock.appid) appid = appid_manager.get_appid() if not appid: html = generate_message_html('404 No usable Appid Exists', u'没有可用appid了,请配置可用的appid') send_response(wfile, 404, body=html.encode('utf-8')) response.close() return else: response.close() continue if response.app_status == 503: logging.warning('APPID %r out of Quota, remove it.', response.ssl_sock.appid) appid_manager.report_out_of_quota(response.ssl_sock.appid) appid = appid_manager.get_appid() if not appid: html = generate_message_html('503 No usable Appid Exists', u'appid流量不足,请增加appid') send_response(wfile, 503, body=html.encode('utf-8')) response.close() return else: response.close() continue if response.app_status < 500: break except GAE_Exception as e: errors.append(e) logging.warn("gae_exception:%s %r", e, url) except Exception as e: errors.append(e) logging.exception('gae_handler.handler %r %s , retry...', e, url) if len(errors) == max_retry: if response and response.app_status >= 500: status = response.app_status headers = dict(response.getheaders()) content = response.read() else: status = 502 headers = {'Content-Type': 'text/html'} content = generate_message_html('502 URLFetch failed', 'Local URLFetch %r failed' % url, '<br>'.join(repr(x) for x in errors)) if response: response.close() send_response(wfile, status, headers, content.encode('utf-8')) logging.warn("GAE %d %s %s", status, method, url) return if response.status == 206: return RangeFetch(method, url, headers, body, response, wfile).fetch() try: wfile.write("HTTP/1.1 %d %s\r\n" % (response.status, response.reason)) response_headers = {} for key, value in response.getheaders(): key = key.title() if key == 'Transfer-Encoding': continue if key in skip_headers: continue response_headers[key] = value if method == "HEAD": if 'X-Head-Content-Length' in response_headers: response_headers['Content-Length'] = response_headers['X-Head-Content-Length'] del response_headers['X-Head-Content-Length'] for key in response_headers: value = response_headers[key] send_header(wfile, key, value) #logging.debug("Head- %s: %s", key, value) wfile.write("\r\n") if len(response.app_msg): logging.warn("APPID error:%d url:%s", response.status, url) wfile.write(response.app_msg) response.close() return content_length = int(response.getheader('Content-Length', 0)) content_range = response.getheader('Content-Range', '') if content_range: start, end, length = tuple(int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)', content_range).group(1, 2, 3)) else: start, end, length = 0, content_length-1, content_length time_start = time.time() send_to_broswer = True while True: data = response.read(config.AUTORANGE_BUFSIZE) if not data and time.time() - time_start > 20: response.close() logging.warn("read timeout t:%d len:%d left:%d %s", (time.time()-time_request)*1000, length, (end-start), url) return data_len = len(data) start += data_len if send_to_broswer: try: ret = wfile.write(data) if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ: logging.debug("send to browser wfile.write ret:%d", ret) ret = wfile.write(data) except Exception as e_b: if e_b[0] in (errno.ECONNABORTED, errno.EPIPE, errno.ECONNRESET) or 'bad write retry' in repr(e_b): logging.warn('gae_handler send to browser return %r %r', e_b, url) else: logging.warn('gae_handler send to browser return %r %r', e_b, url) send_to_broswer = False if start >= end: https_manager.save_ssl_connection_for_reuse(response.ssl_sock) logging.info("GAE t:%d s:%d %d %s", (time.time()-time_request)*1000, length, response.status, url) return except NetWorkIOError as e: if e[0] in (errno.ECONNABORTED, errno.EPIPE) or 'bad write retry' in repr(e): logging.warn("gae_handler err:%r %s ", e, url) else: logging.exception("gae_handler except:%r %s", e, url) except Exception as e: logging.exception("gae_handler except:%r %s", e, url)
def __fetchlet(self, range_queue, data_queue, range_delay_size): headers = dict((k.title(), v) for k, v in self.headers.items()) headers['Connection'] = 'close' while not self._stopped: try: try: start, end, response = range_queue.get(timeout=1) if self.expect_begin < start and data_queue.qsize( ) * self.bufsize + range_delay_size > 30 * 1024 * 1024: range_queue.put((start, end, response)) time.sleep(10) continue headers['Range'] = 'bytes=%d-%d' % (start, end) if not response: response = fetch(self.method, self.url, headers, self.body) except Queue.Empty: continue except Exception as e: xlog.warning("RangeFetch fetch response %r in __fetchlet", e) range_queue.put((start, end, None)) continue if not response: xlog.warning('RangeFetch %s return %r', headers['Range'], response) range_queue.put((start, end, None)) continue if response.app_status != 200: xlog.warning('Range Fetch return %s "%s %s" %s ', response.app_status, self.method, self.url, headers['Range']) if response.app_status == 404: xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid) appid_manager.report_not_exist(response.ssl_sock.appid) appid = appid_manager.get_appid() if not appid: xlog.error("no appid left") self._stopped = True response.close() return if response.app_status == 503: xlog.warning( 'APPID %r out of Quota, remove it temporary.', response.ssl_sock.appid) appid_manager.report_out_of_quota( response.ssl_sock.appid) appid = appid_manager.get_appid() if not appid: xlog.error("no appid left") self._stopped = True response.close() return response.close() range_queue.put((start, end, None)) continue if response.getheader('Location'): self.url = urlparse.urljoin(self.url, response.getheader('Location')) xlog.info('RangeFetch Redirect(%r)', self.url) response.close() range_queue.put((start, end, None)) continue if 200 <= response.status < 300: content_range = response.getheader('Content-Range') if not content_range: xlog.warning( 'RangeFetch "%s %s" return Content-Range=%r: response headers=%r, retry %s-%s', self.method, self.url, content_range, response.getheaders(), start, end) response.close() range_queue.put((start, end, None)) continue content_length = int( response.getheader('Content-Length', 0)) xlog.info('>>>>>>>>>>>>>>> [thread %s] %s %s', threading.currentThread().ident, content_length, content_range) time_last_read = time.time() while start < end + 1: try: data = response.read(self.bufsize) if not data: if time.time() - time_last_read > 20: break else: time.sleep(0.1) continue time_last_read = time.time() data_len = len(data) data_queue.put((start, data)) start += data_len except Exception as e: xlog.warning('RangeFetch "%s %s" %s failed: %s', self.method, self.url, headers['Range'], e) break if start < end + 1: xlog.warning('RangeFetch "%s %s" retry %s-%s', self.method, self.url, start, end) response.close() range_queue.put((start, end, None)) continue https_manager.save_ssl_connection_for_reuse( response.ssl_sock) xlog.info('>>>>>>>>>>>>>>> Successfully reached %d bytes.', start - 1) else: xlog.error('RangeFetch %r return %s', self.url, response.status) response.close() range_queue.put((start, end, None)) continue except StandardError as e: xlog.exception('RangeFetch._fetchlet error:%s', e) raise
def handler(method, url, headers, body, wfile): time_request = time.time() headers = clean_empty_header(headers) errors = [] response = None while True: if time.time() - time_request > 30: #time out return return_fail_message(wfile) try: response = fetch(method, url, headers, body) if response.app_status != 200: xlog.warn("fetch gae status:%s url:%s", response.app_status, url) if response.app_status == 404: xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid) appid_manager.report_not_exist(response.ssl_sock.appid) appid = appid_manager.get_appid() if not appid: html = generate_message_html('404 No usable Appid Exists', u'没有可用appid了,请配置可用的appid') send_response(wfile, 404, body=html.encode('utf-8')) response.close() return else: response.close() continue if response.app_status == 403 or response.app_status == 405: #Method not allowed # google have changed from gws to gvs, need to remove. xlog.warning('405 Method not allowed. remove %s ', response.ssl_sock.ip) # some ip can connect, and server type is gws # but can't use as GAE server # so we need remove it immediately google_ip.report_connect_fail(response.ssl_sock.ip, force_remove=True) response.close() continue if response.app_status == 503: xlog.warning('APPID %r out of Quota, remove it.', response.ssl_sock.appid) appid_manager.report_out_of_quota(response.ssl_sock.appid) appid = appid_manager.get_appid() if not appid: html = generate_message_html('503 No usable Appid Exists', u'appid流量不足,请增加appid') send_response(wfile, 503, body=html.encode('utf-8')) response.close() return else: response.close() continue if response.app_status < 500: break except GAE_Exception as e: errors.append(e) xlog.warn("gae_exception:%r %s", e, url) except Exception as e: errors.append(e) xlog.exception('gae_handler.handler %r %s , retry...', e, url) if response.status == 206: return RangeFetch(method, url, headers, body, response, wfile).fetch() try: wfile.write("HTTP/1.1 %d %s\r\n" % (response.status, response.reason)) response_headers = {} for key, value in response.getheaders(): key = key.title() if key == 'Transfer-Encoding': #http://en.wikipedia.org/wiki/Chunked_transfer_encoding continue if key in skip_headers: continue response_headers[key] = value if 'X-Head-Content-Length' in response_headers: if method == "HEAD": response_headers['Content-Length'] = response_headers[ 'X-Head-Content-Length'] del response_headers['X-Head-Content-Length'] send_to_browser = True try: for key in response_headers: value = response_headers[key] send_header(wfile, key, value) #logging.debug("Head- %s: %s", key, value) wfile.write("\r\n") except Exception as e: send_to_browser = False xlog.warn("gae_handler.handler send response fail. t:%d e:%r %s", time.time() - time_request, e, url) if len(response.app_msg): xlog.warn("APPID error:%d url:%s", response.status, url) wfile.write(response.app_msg) response.close() return content_length = int(response.getheader('Content-Length', 0)) content_range = response.getheader('Content-Range', '') if content_range: start, end, length = tuple( int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)', content_range).group(1, 2, 3)) else: start, end, length = 0, content_length - 1, content_length last_read_time = time.time() while True: if start > end: https_manager.save_ssl_connection_for_reuse(response.ssl_sock) xlog.info("GAE t:%d s:%d %d %s", (time.time() - time_request) * 1000, length, response.status, url) return data = response.read(config.AUTORANGE_BUFSIZE) if not data: if time.time() - last_read_time > 20: response.close() xlog.warn("read timeout t:%d len:%d left:%d %s", (time.time() - time_request) * 1000, length, (end - start), url) return else: time.sleep(0.1) continue last_read_time = time.time() data_len = len(data) start += data_len if send_to_browser: try: ret = wfile.write(data) if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ: xlog.debug("send to browser wfile.write ret:%d", ret) ret = wfile.write(data) except Exception as e_b: if e_b[0] in (errno.ECONNABORTED, errno.EPIPE, errno.ECONNRESET ) or 'bad write retry' in repr(e_b): xlog.warn('gae_handler send to browser return %r %r', e_b, url) else: xlog.warn('gae_handler send to browser return %r %r', e_b, url) send_to_browser = False except NetWorkIOError as e: time_except = time.time() time_cost = time_except - time_request if e[0] in (errno.ECONNABORTED, errno.EPIPE) or 'bad write retry' in repr(e): xlog.warn("gae_handler err:%r time:%d %s ", e, time_cost, url) else: xlog.exception("gae_handler except:%r %s", e, url) except Exception as e: xlog.exception("gae_handler except:%r %s", e, url)
def do_AGENT(self): """GAE http urlfetch""" request_headers = dict((k.title(), v) for k, v in self.headers.items()) host = request_headers.get('Host', '') path = self.parsed_url.path range_in_query = 'range=' in self.parsed_url.query special_range = (any(x(host) for x in config.AUTORANGE_HOSTS_MATCH) or path.endswith(config.AUTORANGE_ENDSWITH)) and not path.endswith(config.AUTORANGE_NOENDSWITH) if 'Range' in request_headers: m = re.search(r'bytes=(\d+)-', request_headers['Range']) start = int(m.group(1) if m else 0) request_headers['Range'] = 'bytes=%d-%d' % (start, start+config.AUTORANGE_MAXSIZE-1) logging.info('autorange range=%r match url=%r', request_headers['Range'], self.path) elif not range_in_query and special_range: logging.info('Found [autorange]endswith match url=%r', self.path) m = re.search(r'bytes=(\d+)-', request_headers.get('Range', '')) start = int(m.group(1) if m else 0) request_headers['Range'] = 'bytes=%d-%d' % (start, start+config.AUTORANGE_MAXSIZE-1) payload = b'' if 'Content-Length' in request_headers: try: payload = self.rfile.read(int(request_headers.get('Content-Length', 0))) except NetWorkIOError as e: logging.error('handle_method_urlfetch read payload failed:%s', e) return response = None errors = [] headers_sent = False for retry in range(config.FETCHMAX_LOCAL): try: content_length = 0 kwargs = {} # TODO: test validate = 1 kwargs['validate'] = 0 time_start = time.time() response = gae_urlfetch(self.command, self.path, request_headers, payload, **kwargs) time_stop = time.time() time_cost = int((time_stop - time_start) * 1000) if not response: if retry >= config.FETCHMAX_LOCAL-1: html = generate_message_html('502 URLFetch failed', 'Local URLFetch %r failed' % self.path, str(errors)) self.wfile.write(b'HTTP/1.0 502\r\nContent-Type: text/html\r\n\r\n' + html.encode('utf-8')) logging.warning('GET no response %s ', self.path) return else: continue # appid not exists, try remove it from appid if response.app_status == 404: logging.warning('APPID %r not exists, remove it.', response.ssl_sock.appid) appid_manager.report_not_exist(response.ssl_sock.appid) appid = appid_manager.get_appid() if not appid: html = generate_message_html('404 No usable Appid Exists', u'没有可用appid了,请配置可用的appid') self.wfile.write(b'HTTP/1.0 404\r\nContent-Type: text/html\r\n\r\n' + html.encode('utf-8')) response.close() return else: continue # appid over qouta, switch to next appid if response.app_status == 503: logging.warning('APPID %r out of Quota, remove it.', response.ssl_sock.appid) appid_manager.report_out_of_quota(response.ssl_sock.appid) appid = appid_manager.get_appid() if not appid: html = generate_message_html('503 No usable Appid Exists', u'appid流量不足,请增加appid') self.wfile.write(b'HTTP/1.0 503\r\nContent-Type: text/html\r\n\r\n' + html.encode('utf-8')) response.close() return else: continue # 500 is Web server internal err if response.app_status == 500 and range_in_query and special_range: logging.warning('500 with range in query') #, need trying another APPID? response.close() continue if response.app_status == 501: deploy_url = "http://127.0.0.1:8085/?module=goagent&menu=deploy" message = u'请重新部署服务端: <a href="%s">%s</a>' % (deploy_url, deploy_url) html = generate_message_html('Please deploy your new server', message) self.wfile.write(b'HTTP/1.0 501\r\nContent-Type: text/html\r\n\r\n' + html.encode('utf-8')) logging.warning('501 Please deploy your new server') #, need trying another APPID? response.close() return if response.app_status != 200 and retry == config.FETCHMAX_LOCAL-1: logging.warn('GAE %s %s status:%s', self.command, self.path, response.status) self.wfile.write(('HTTP/1.1 %s\r\n%s\r\n' % (response.status, ''.join('%s: %s\r\n' % (k.title(), v) for k, v in response.getheaders() if k.title() != 'Transfer-Encoding')))) self.wfile.write(response.read()) response.close() return # first response, has no retry. if not headers_sent: data_size = response.getheader('Content-Length', '0') download_speed = int(data_size) * 1000 / time_cost logging.info('"GAE t:%d speed:%d len:%s status:%s %s %s HTTP/1.1"', time_cost, download_speed, response.getheader('Content-Length', '-'), response.status, self.command, self.path) if response.status == 206: # 206 means "Partial Content" rangefetch = RangeFetch(self.wfile, response, self.command, self.path, self.headers, payload, maxsize=config.AUTORANGE_MAXSIZE, bufsize=config.AUTORANGE_BUFSIZE, waitsize=config.AUTORANGE_WAITSIZE, threads=config.AUTORANGE_THREADS) return rangefetch.fetch() if response.getheader('Set-Cookie'): response.msg['Set-Cookie'] = self.normcookie(response.getheader('Set-Cookie')) if response.getheader('Content-Disposition') and '"' not in response.getheader('Content-Disposition'): response.msg['Content-Disposition'] = self.normattachment(response.getheader('Content-Disposition')) headers_data = 'HTTP/1.1 %s\r\n%s\r\n' % (response.status, ''.join('%s: %s\r\n' % (k.title(), v) for k, v in response.getheaders() if k.title() != 'Transfer-Encoding')) #logging.debug('headers_data=%s', headers_data) #self.wfile.write(headers_data.encode() if bytes is not str else headers_data) self.wfile.write(headers_data) headers_sent = True content_length = int(response.getheader('Content-Length', 0)) content_range = response.getheader('Content-Range', '') accept_ranges = response.getheader('Accept-Ranges', 'none') if content_range: start, end, length = tuple(int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)', content_range).group(1, 2, 3)) else: start, end, length = 0, content_length-1, content_length if content_length == 0: https_manager.save_ssl_connection_for_reuse(response.ssl_sock) return send_to_broswer = True while True: data = response.read(8192) #TODO: loop read until timeout or except. if not data: response.close() return data_len = len(data) start += data_len if send_to_broswer: try: ret = self.wfile.write(data) if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ: logging.debug("self.wfile.write ret:%d", ret) ret = self.wfile.write(data) except Exception as e_b: logging.exception('GAEProxyHandler.do_METHOD_AGENT send to browser %r return %r', self.path, e_b) send_to_broswer = False if start >= end: https_manager.save_ssl_connection_for_reuse(response.ssl_sock) return except Exception as e: logging.exception('GAEProxyHandler.do_METHOD_AGENT %r return %r', self.path, e) errors.append(e) if response: response.close() if e.args[0] in (errno.ECONNABORTED, errno.EPIPE, errno.ECONNRESET): #logging.debug('GAEProxyHandler.do_METHOD_AGENT return %r', e) pass elif e.args[0] == errno.ETIMEDOUT or isinstance(e.args[0], str) and 'timed out' in e.args[0]: if content_length and accept_ranges == 'bytes': # we can retry range fetch here logging.warn('GAEProxyHandler.do_METHOD_AGENT timed out, content_length=%r, url=%r, try again', content_length, self.path) self.headers['Range'] = 'bytes=%d-%d' % (start, end) elif isinstance(e, NetWorkIOError) and 'bad write retry' in e.args[-1]: logging.warn('GAEProxyHandler.do_METHOD_AGENT return %r, abort. url=%r ', e, self.path) return else: logging.exception('GAEProxyHandler.do_METHOD_AGENT %r return %r', self.path, e) #IOError(9, 'Bad file descriptor'), int(e.args[0])
def handler(method, url, headers, body, wfile): time_request = time.time() errors = [] response = None for i in xrange(max_retry): try: response = fetch(method, url, headers, body) if response.app_status != 200: logging.debug("fetch gae status:%d url:%s", response.app_status, url) if response.app_status == 404: logging.warning('APPID %r not exists, remove it.', response.ssl_sock.appid) appid_manager.report_not_exist(response.ssl_sock.appid) appid = appid_manager.get_appid() if not appid: html = generate_message_html('404 No usable Appid Exists', u'没有可用appid了,请配置可用的appid') send_response(wfile, 404, body=html.encode('utf-8')) response.close() return else: response.close() continue if response.app_status == 503: logging.warning('APPID %r out of Quota, remove it.', response.ssl_sock.appid) appid_manager.report_out_of_quota(response.ssl_sock.appid) appid = appid_manager.get_appid() if not appid: html = generate_message_html('503 No usable Appid Exists', u'appid流量不足,请增加appid') send_response(wfile, 503, body=html.encode('utf-8')) response.close() return else: response.close() continue if response.app_status < 500: break except GAE_Exception as e: errors.append(e) logging.warn("gae_exception:%s %r", e, url) except Exception as e: errors.append(e) logging.exception('gae_handler.handler %r %s , retry...', e, url) if len(errors) == max_retry: if response and response.app_status >= 500: status = response.app_status headers = dict(response.getheaders()) content = response.read() else: status = 502 headers = {'Content-Type': 'text/html'} content = generate_message_html( '502 URLFetch failed', 'Local URLFetch %r failed' % url, '<br>'.join(repr(x) for x in errors)) if response: response.close() send_response(wfile, status, headers, content.encode('utf-8')) logging.warn("GAE %d %s %s", status, method, url) return if response.status == 206: return RangeFetch(method, url, headers, body, response, wfile).fetch() try: wfile.write("HTTP/1.1 %d\r\n" % response.status) response_headers = {} for key, value in response.getheaders(): key = key.title() if key == 'Transfer-Encoding': continue if key in skip_headers: continue response_headers[key] = value if method == "HEAD": if 'X-Head-Content-Length' in response_headers: response_headers['Content-Length'] = response_headers[ 'X-Head-Content-Length'] del response_headers['X-Head-Content-Length'] for key in response_headers: value = response_headers[key] send_header(wfile, key, value) #logging.debug("Head- %s: %s", key, value) wfile.write("\r\n") if len(response.app_msg): logging.warn("APPID error:%d url:%s", response.status, url) wfile.write(response.app_msg) response.close() return content_length = int(response.getheader('Content-Length', 0)) content_range = response.getheader('Content-Range', '') if content_range: start, end, length = tuple( int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)', content_range).group(1, 2, 3)) else: start, end, length = 0, content_length - 1, content_length time_start = time.time() send_to_broswer = True while True: data = response.read(config.AUTORANGE_BUFSIZE) if not data and time.time() - time_start > 20: response.close() logging.warn("read timeout t:%d len:%d left:%d %s", (time.time() - time_request) * 1000, length, (end - start), url) return data_len = len(data) start += data_len if send_to_broswer: try: ret = wfile.write(data) if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ: logging.debug("send to browser wfile.write ret:%d", ret) ret = wfile.write(data) except Exception as e_b: if e_b[0] in (errno.ECONNABORTED, errno.EPIPE, errno.ECONNRESET ) or 'bad write retry' in repr(e_b): logging.warn( 'gae_handler send to browser return %r %r', e_b, url) else: logging.warn( 'gae_handler send to browser return %r %r', e_b, url) send_to_broswer = False if start >= end: https_manager.save_ssl_connection_for_reuse(response.ssl_sock) logging.info("GAE t:%d s:%d %d %s", (time.time() - time_request) * 1000, length, response.status, url) return except NetWorkIOError as e: if e[0] in (errno.ECONNABORTED, errno.EPIPE) or 'bad write retry' in repr(e): logging.warn("gae_handler err:%r %s ", e, url) else: logging.exception("gae_handler except:%r %s", e, url) except Exception as e: logging.exception("gae_handler except:%r %s", e, url)
def handler(method, url, headers, body, wfile): time_request = time.time() errors = [] response = None while True: if time.time() - time_request > 90: #time out return return_fail_message(wfile) try: response = fetch(method, url, headers, body) if response.app_status != 200: logging.warn("fetch gae status:%s url:%s", response.app_status, url) if response.app_status == 404: logging.warning('APPID %r not exists, remove it.', response.ssl_sock.appid) appid_manager.report_not_exist(response.ssl_sock.appid) appid = appid_manager.get_appid() if not appid: html = generate_message_html('404 No usable Appid Exists', u'没有可用appid了,请配置可用的appid') send_response(wfile, 404, body=html.encode('utf-8')) response.close() return else: response.close() continue if response.app_status == 403 or response.app_status == 405: #Method not allowed # google have changed from gws to gvs, need to remove. logging.warning('405 Method not allowed. remove %s ', response.ssl_sock.ip) google_ip.report_connect_fail(response.ssl_sock.ip, force_remove=True) response.close() continue if response.app_status == 503: logging.warning('APPID %r out of Quota, remove it.', response.ssl_sock.appid) appid_manager.report_out_of_quota(response.ssl_sock.appid) appid = appid_manager.get_appid() if not appid: html = generate_message_html('503 No usable Appid Exists', u'appid流量不足,请增加appid') send_response(wfile, 503, body=html.encode('utf-8')) response.close() return else: response.close() continue if response.app_status < 500: break except GAE_Exception as e: errors.append(e) logging.warn("gae_exception:%r %s", e, url) except Exception as e: errors.append(e) logging.exception('gae_handler.handler %r %s , retry...', e, url) if response.status == 206: return RangeFetch(method, url, headers, body, response, wfile).fetch() try: wfile.write("HTTP/1.1 %d %s\r\n" % (response.status, response.reason)) response_headers = {} for key, value in response.getheaders(): key = key.title() if key == 'Transfer-Encoding': #http://en.wikipedia.org/wiki/Chunked_transfer_encoding continue if key in skip_headers: continue response_headers[key] = value if 'X-Head-Content-Length' in response_headers: if method == "HEAD": response_headers['Content-Length'] = response_headers['X-Head-Content-Length'] del response_headers['X-Head-Content-Length'] for key in response_headers: value = response_headers[key] send_header(wfile, key, value) #logging.debug("Head- %s: %s", key, value) wfile.write("\r\n") if len(response.app_msg): logging.warn("APPID error:%d url:%s", response.status, url) wfile.write(response.app_msg) response.close() return content_length = int(response.getheader('Content-Length', 0)) content_range = response.getheader('Content-Range', '') if content_range: start, end, length = tuple(int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)', content_range).group(1, 2, 3)) else: start, end, length = 0, content_length-1, content_length send_to_broswer = True while True: time_start = time.time() data = response.read(config.AUTORANGE_BUFSIZE) if not data and time.time() - time_start > 20: response.close() logging.warn("read timeout t:%d len:%d left:%d %s", (time.time()-time_request)*1000, length, (end-start), url) return data_len = len(data) start += data_len if send_to_broswer: try: ret = wfile.write(data) if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ: logging.debug("send to browser wfile.write ret:%d", ret) ret = wfile.write(data) except Exception as e_b: if e_b[0] in (errno.ECONNABORTED, errno.EPIPE, errno.ECONNRESET) or 'bad write retry' in repr(e_b): logging.warn('gae_handler send to browser return %r %r', e_b, url) else: logging.warn('gae_handler send to browser return %r %r', e_b, url) send_to_broswer = False if start >= end: https_manager.save_ssl_connection_for_reuse(response.ssl_sock) logging.info("GAE t:%d s:%d %d %s", (time.time()-time_request)*1000, length, response.status, url) return except NetWorkIOError as e: time_except = time.time() time_cost = time_except - time_request if e[0] in (errno.ECONNABORTED, errno.EPIPE) or 'bad write retry' in repr(e): logging.warn("gae_handler err:%r time:%d %s ", e, time_cost, url) else: logging.exception("gae_handler except:%r %s", e, url) except Exception as e: logging.exception("gae_handler except:%r %s", e, url)
def do_AGENT(self): """GAE http urlfetch""" request_headers = dict((k.title(), v) for k, v in self.headers.items()) host = request_headers.get('Host', '') path = self.parsed_url.path range_in_query = 'range=' in self.parsed_url.query special_range = (any(x(host) for x in config.AUTORANGE_HOSTS_MATCH) or path.endswith(config.AUTORANGE_ENDSWITH) ) and not path.endswith(config.AUTORANGE_NOENDSWITH) if 'Range' in request_headers: m = re.search(r'bytes=(\d+)-', request_headers['Range']) start = int(m.group(1) if m else 0) request_headers['Range'] = 'bytes=%d-%d' % ( start, start + config.AUTORANGE_MAXSIZE - 1) logging.info('autorange range=%r match url=%r', request_headers['Range'], self.path) elif not range_in_query and special_range: logging.info('Found [autorange]endswith match url=%r', self.path) m = re.search(r'bytes=(\d+)-', request_headers.get('Range', '')) start = int(m.group(1) if m else 0) request_headers['Range'] = 'bytes=%d-%d' % ( start, start + config.AUTORANGE_MAXSIZE - 1) payload = b'' if 'Content-Length' in request_headers: try: payload = self.rfile.read( int(request_headers.get('Content-Length', 0))) except NetWorkIOError as e: logging.error('handle_method_urlfetch read payload failed:%s', e) return response = None errors = [] headers_sent = False for retry in range(config.FETCHMAX_LOCAL): try: content_length = 0 kwargs = {} # TODO: test validate = 1 kwargs['validate'] = 0 time_start = time.time() response = gae_urlfetch(self.command, self.path, request_headers, payload, **kwargs) time_stop = time.time() time_cost = int((time_stop - time_start) * 1000) if not response: if retry >= config.FETCHMAX_LOCAL - 1: html = generate_message_html( '502 URLFetch failed', 'Local URLFetch %r failed' % self.path, str(errors)) self.wfile.write( b'HTTP/1.0 502\r\nContent-Type: text/html\r\n\r\n' + html.encode('utf-8')) logging.warning('GET no response %s ', self.path) return else: continue # appid not exists, try remove it from appid if response.app_status == 404: logging.warning('APPID %r not exists, remove it.', response.ssl_sock.appid) appid_manager.report_not_exist(response.ssl_sock.appid) appid = appid_manager.get_appid() if not appid: html = generate_message_html( '404 No usable Appid Exists', u'没有可用appid了,请配置可用的appid') self.wfile.write( b'HTTP/1.0 404\r\nContent-Type: text/html\r\n\r\n' + html.encode('utf-8')) response.close() return else: continue # appid over qouta, switch to next appid if response.app_status == 503: logging.warning('APPID %r out of Quota, remove it.', response.ssl_sock.appid) appid_manager.report_out_of_quota(response.ssl_sock.appid) appid = appid_manager.get_appid() if not appid: html = generate_message_html( '503 No usable Appid Exists', u'appid流量不足,请增加appid') self.wfile.write( b'HTTP/1.0 503\r\nContent-Type: text/html\r\n\r\n' + html.encode('utf-8')) response.close() return else: continue # 500 is Web server internal err if response.app_status == 500 and range_in_query and special_range: logging.warning('500 with range in query' ) #, need trying another APPID? response.close() continue if response.app_status == 501: deploy_url = "http://127.0.0.1:8085/?module=goagent&menu=deploy" message = u'请重新部署服务端: <a href="%s">%s</a>' % (deploy_url, deploy_url) html = generate_message_html( 'Please deploy your new server', message) self.wfile.write( b'HTTP/1.0 501\r\nContent-Type: text/html\r\n\r\n' + html.encode('utf-8')) logging.warning('501 Please deploy your new server' ) #, need trying another APPID? response.close() return if response.app_status != 200 and retry == config.FETCHMAX_LOCAL - 1: logging.warn('GAE %s %s status:%s', self.command, self.path, response.status) self.wfile.write( ('HTTP/1.1 %s\r\n%s\r\n' % (response.status, ''.join( '%s: %s\r\n' % (k.title(), v) for k, v in response.getheaders() if k.title() != 'Transfer-Encoding')))) self.wfile.write(response.read()) response.close() return # first response, has no retry. if not headers_sent: data_size = response.getheader('Content-Length', '0') download_speed = int(data_size) * 1000 / time_cost logging.info( '"GAE t:%d speed:%d len:%s status:%s %s %s HTTP/1.1"', time_cost, download_speed, response.getheader('Content-Length', '-'), response.status, self.command, self.path) if response.status == 206: # 206 means "Partial Content" rangefetch = RangeFetch( self.wfile, response, self.command, self.path, self.headers, payload, maxsize=config.AUTORANGE_MAXSIZE, bufsize=config.AUTORANGE_BUFSIZE, waitsize=config.AUTORANGE_WAITSIZE, threads=config.AUTORANGE_THREADS) return rangefetch.fetch() if response.getheader('Set-Cookie'): response.msg['Set-Cookie'] = self.normcookie( response.getheader('Set-Cookie')) if response.getheader('Content-Disposition' ) and '"' not in response.getheader( 'Content-Disposition'): response.msg[ 'Content-Disposition'] = self.normattachment( response.getheader('Content-Disposition')) headers_data = 'HTTP/1.1 %s\r\n%s\r\n' % ( response.status, ''.join( '%s: %s\r\n' % (k.title(), v) for k, v in response.getheaders() if k.title() != 'Transfer-Encoding')) #logging.debug('headers_data=%s', headers_data) #self.wfile.write(headers_data.encode() if bytes is not str else headers_data) self.wfile.write(headers_data) headers_sent = True content_length = int(response.getheader('Content-Length', 0)) content_range = response.getheader('Content-Range', '') accept_ranges = response.getheader('Accept-Ranges', 'none') if content_range: start, end, length = tuple( int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)', content_range).group(1, 2, 3)) else: start, end, length = 0, content_length - 1, content_length if content_length == 0: https_manager.save_ssl_connection_for_reuse( response.ssl_sock) return send_to_broswer = True while True: data = response.read( 8192) #TODO: loop read until timeout or except. if not data: response.close() return data_len = len(data) start += data_len if send_to_broswer: try: ret = self.wfile.write(data) if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ: logging.debug("self.wfile.write ret:%d", ret) ret = self.wfile.write(data) except Exception as e_b: logging.exception( 'GAEProxyHandler.do_METHOD_AGENT send to browser %r return %r', self.path, e_b) send_to_broswer = False if start >= end: https_manager.save_ssl_connection_for_reuse( response.ssl_sock) return except Exception as e: logging.exception( 'GAEProxyHandler.do_METHOD_AGENT %r return %r', self.path, e) errors.append(e) if response: response.close() if e.args[0] in (errno.ECONNABORTED, errno.EPIPE, errno.ECONNRESET): #logging.debug('GAEProxyHandler.do_METHOD_AGENT return %r', e) pass elif e.args[0] == errno.ETIMEDOUT or isinstance( e.args[0], str) and 'timed out' in e.args[0]: if content_length and accept_ranges == 'bytes': # we can retry range fetch here logging.warn( 'GAEProxyHandler.do_METHOD_AGENT timed out, content_length=%r, url=%r, try again', content_length, self.path) self.headers['Range'] = 'bytes=%d-%d' % (start, end) elif isinstance( e, NetWorkIOError) and 'bad write retry' in e.args[-1]: logging.warn( 'GAEProxyHandler.do_METHOD_AGENT return %r, abort. url=%r ', e, self.path) return else: logging.exception( 'GAEProxyHandler.do_METHOD_AGENT %r return %r', self.path, e) #IOError(9, 'Bad file descriptor'), int(e.args[0])
def request_gae_proxy(method, url, headers, body, timeout=60, retry=True): headers = dict(headers) # make retry and time out time_request = time.time() # GAE urlfetch will not decode br if Accept-Encoding include gzip accept_encoding = headers.get("Accept-Encoding", "") if "br" in accept_encoding: accept_br_encoding = True xlog.debug("accept_br_encoding for %s", url) else: accept_br_encoding = False host = headers.get("Host", "") if not host: parsed_url = urlparse.urlparse(url) host = parsed_url.hostname accept_codes = accept_encoding.replace(" ", "").split(",") if not accept_br_encoding: if "gzip" in accept_encoding and host in config.br_sites: accept_codes.remove("gzip") if "br" not in accept_codes: accept_codes.append("br") accept_code_str = ",".join(accept_codes) if accept_code_str: headers["Accept-Encoding"] = accept_code_str else: del headers["Accept-Encoding"] request_headers, request_body = pack_request(method, url, headers, body) error_msg = [] while True: if time.time() - time_request > timeout: raise GAE_Exception(600, b"".join(error_msg)) if not retry and error_msg: raise GAE_Exception(600, b"".join(error_msg)) try: response = request_gae_server(request_headers, request_body, url, timeout) response = unpack_response(response) # xlog.debug("accept:%s content-encoding:%s url:%s", accept_encoding, # response.headers.get("Content-Encoding", ""), url) if not accept_br_encoding: # if gzip in Accept-Encoding, br will not decode in urlfetch # else, urlfetch in GAE will auto decode br, but return br in Content-Encoding if response.headers.get("Content-Encoding", "") == "br": # GAE urlfetch always return br in content-encoding even have decoded it. del response.headers["Content-Encoding"] # xlog.debug("remove br from Content-Encoding, %s", url) if host not in config.br_sites: br_sites = list(config.br_sites) br_sites.append(host) config.br_sites = tuple(br_sites) xlog.warn("Add %s to br_sites", host) if response.app_msg: xlog.warn("server app return fail, status:%d", response.app_status) # if len(response.app_msg) < 2048: # xlog.warn('app_msg:%s', cgi.escape(response.app_msg)) if response.app_status == 510: # reach 80% of traffic today # disable for get big file. appid_manager.report_out_of_quota(response.ssl_sock.appid) response.worker.close( "appid out of quota:%s" % response.ssl_sock.appid) continue return response except GAE_Exception as e: err_msg = "gae_exception:%r %s" % (e, url) error_msg.append(err_msg) xlog.warn("gae_exception:%r %s", e, url) if e.message == '605:status:500': raise e except Exception as e: err_msg = 'gae_handler.handler %r %s , retry...' % (e, url) error_msg.append(err_msg) xlog.exception('gae_handler.handler %r %s , retry...', e, url)
def handler(method, url, headers, body, wfile): time_request = time.time() errors = [] response = None while True: if time.time() - time_request > 30: # time out html = generate_message_html("504 GoAgent Proxy Time out", u"GoAgent代理处理超时,请查看日志!") send_response(wfile, 504, body=html.encode("utf-8")) return try: response = fetch(method, url, headers, body) if response.app_status != 200: logging.debug("fetch gae status:%s url:%s", response.app_status, url) if response.app_status == 404: logging.warning("APPID %r not exists, remove it.", response.ssl_sock.appid) appid_manager.report_not_exist(response.ssl_sock.appid) appid = appid_manager.get_appid() if not appid: html = generate_message_html("404 No usable Appid Exists", u"没有可用appid了,请配置可用的appid") send_response(wfile, 404, body=html.encode("utf-8")) response.close() return else: response.close() continue if response.app_status == 405: # Method not allowed logging.warning("405 Method not allowed. remove %s ", response.ssl_sock.ip) google_ip.report_connect_fail(response.ssl_sock.ip, force_remove=True) response.close() continue if response.app_status == 503: logging.warning("APPID %r out of Quota, remove it.", response.ssl_sock.appid) appid_manager.report_out_of_quota(response.ssl_sock.appid) appid = appid_manager.get_appid() if not appid: html = generate_message_html("503 No usable Appid Exists", u"appid流量不足,请增加appid") send_response(wfile, 503, body=html.encode("utf-8")) response.close() return else: response.close() continue if response.app_status < 500: break except GAE_Exception as e: errors.append(e) logging.warn("gae_exception:%s %r", e, url) except Exception as e: errors.append(e) logging.exception("gae_handler.handler %r %s , retry...", e, url) if len(errors) == max_retry: if response and response.app_status >= 500: status = response.app_status headers = dict(response.getheaders()) content = response.read() else: status = 502 headers = {"Content-Type": "text/html"} content = generate_message_html( "502 URLFetch failed", "Local URLFetch %r failed" % url, "<br>".join(repr(x) for x in errors) ) if response: response.close() send_response(wfile, status, headers, content.encode("utf-8")) logging.warn("GAE %d %s %s", status, method, url) return if response.status == 206: return RangeFetch(method, url, headers, body, response, wfile).fetch() try: wfile.write("HTTP/1.1 %d %s\r\n" % (response.status, response.reason)) response_headers = {} for key, value in response.getheaders(): key = key.title() if key == "Transfer-Encoding": # http://en.wikipedia.org/wiki/Chunked_transfer_encoding continue if key in skip_headers: continue response_headers[key] = value if "X-Head-Content-Length" in response_headers: if method == "HEAD": response_headers["Content-Length"] = response_headers["X-Head-Content-Length"] del response_headers["X-Head-Content-Length"] for key in response_headers: value = response_headers[key] send_header(wfile, key, value) # logging.debug("Head- %s: %s", key, value) wfile.write("\r\n") if len(response.app_msg): logging.warn("APPID error:%d url:%s", response.status, url) wfile.write(response.app_msg) response.close() return content_length = int(response.getheader("Content-Length", 0)) content_range = response.getheader("Content-Range", "") if content_range: start, end, length = tuple( int(x) for x in re.search(r"bytes (\d+)-(\d+)/(\d+)", content_range).group(1, 2, 3) ) else: start, end, length = 0, content_length - 1, content_length time_start = time.time() send_to_broswer = True while True: data = response.read(config.AUTORANGE_BUFSIZE) if not data and time.time() - time_start > 20: response.close() logging.warn( "read timeout t:%d len:%d left:%d %s", (time.time() - time_request) * 1000, length, (end - start), url, ) return data_len = len(data) start += data_len if send_to_broswer: try: ret = wfile.write(data) if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ: logging.debug("send to browser wfile.write ret:%d", ret) ret = wfile.write(data) except Exception as e_b: if e_b[0] in (errno.ECONNABORTED, errno.EPIPE, errno.ECONNRESET) or "bad write retry" in repr(e_b): logging.warn("gae_handler send to browser return %r %r", e_b, url) else: logging.warn("gae_handler send to browser return %r %r", e_b, url) send_to_broswer = False if start >= end: https_manager.save_ssl_connection_for_reuse(response.ssl_sock) logging.info("GAE t:%d s:%d %d %s", (time.time() - time_request) * 1000, length, response.status, url) return except NetWorkIOError as e: if e[0] in (errno.ECONNABORTED, errno.EPIPE) or "bad write retry" in repr(e): logging.warn("gae_handler err:%r %s ", e, url) else: logging.exception("gae_handler except:%r %s", e, url) except Exception as e: logging.exception("gae_handler except:%r %s", e, url)