def head_request(ssl_sock):
if ssl_sock.host == '':
ssl_sock.appid = appid_manager.get_appid()
if not ssl_sock.appid:
logging.error("no appid can use")
return False
host = ssl_sock.appid + ".appspot.com"
ssl_sock.host = host
else:
host = ssl_sock.host
logging.debug("head request %s", host)
request_data = 'HEAD /_gh/ HTTP/1.1\r\nHost: %s\r\n\r\n' % host
try:
ssl_sock.settimeout(2)
ssl_sock.send(request_data.encode())
response = httplib.HTTPResponse(ssl_sock, buffering=True)
response.begin()
status = response.status
if status != 200:
logging.debug("app head fail status:%d", status)
raise Exception("app check fail")
content = response.read()
return True
except Exception as e:
logging.debug("head request fail:%r", e)
return False
finally:
response.close()
def request(headers={}, payload=None):
max_retry = 3
for i in range(max_retry):
ssl_sock = None
try:
ssl_sock = https_manager.create_ssl_connection()
if not ssl_sock:
logging.debug('create_ssl_connection fail')
continue
if ssl_sock.host == '':
ssl_sock.appid = appid_manager.get_appid()
if not ssl_sock.appid:
raise GAE_Exception(1, "no appid can use")
headers['Host'] = ssl_sock.appid + ".appspot.com"
ssl_sock.host = headers['Host']
else:
headers['Host'] = ssl_sock.host
response = _request(ssl_sock, headers, payload)
if not response:
continue
response.ssl_sock = ssl_sock
return response
except Exception as e:
logging.warn('request failed:%s', e)
if ssl_sock:
ssl_sock.close()
raise GAE_Exception(2, "try max times")
def on_ssl_created_cb(self, ssl_sock, check_free_worke=True):
if not ssl_sock:
raise Exception("on_ssl_created_cb ssl_sock None")
appid = appid_manager.get_appid()
if not appid:
time.sleep(60)
ssl_sock.close()
raise http_common.GAE_Exception(1, "no appid can use")
ssl_sock.appid = appid
ssl_sock.host = ssl_sock.appid + ".appspot.com"
if ssl_sock.h2:
worker = HTTP2_worker(ssl_sock, self.close_cb, self.retry_task_cb, self._on_worker_idle_cb, self.log_debug_data)
self.h2_num += 1
else:
worker = HTTP1_worker(ssl_sock, self.close_cb, self.retry_task_cb, self._on_worker_idle_cb, self.log_debug_data)
self.h1_num += 1
self.workers.append(worker)
self.wait_a_worker_cv.notify()
if check_free_worke:
self.check_free_worker()
def test_app_head(ssl_sock, ip):
appid = appid_manager.get_appid()
request_data = 'GET / HTTP/1.1\r\nHost: %s.appspot.com\r\n\r\n' % appid
time_start = time.time()
ssl_sock.send(request_data.encode())
response = httplib.HTTPResponse(ssl_sock, buffering=True)
try:
response.begin()
status = response.status
if status != 200:
xlog.debug("app check %s status:%d", ip, status)
raise Exception("app check fail")
content = response.read()
if "GoAgent" not in content:
xlog.debug("app check %s content:%s", ip, content)
raise Exception("content fail")
except Exception as e:
xlog.exception("test_app_head except:%r", e)
return False
finally:
response.close()
time_stop = time.time()
time_cost = (time_stop - time_start)*1000
xlog.debug("app check time:%d", time_cost)
return True
def test_gae_ip(ip):
try:
ssl_sock = connect_ssl(ip, timeout=max_timeout, openssl_context=openssl_context)
get_ssl_cert_domain(ssl_sock)
appid = appid_manager.get_appid()
check_appid(ssl_sock, appid)
return ssl_sock
except Exception as e:
#xlog.exception("test_gae_ip %s e:%r",ip, e)
return False
def head_request(self, ssl_sock):
if ssl_sock.host == '':
ssl_sock.appid = appid_manager.get_appid()
if not ssl_sock.appid:
xlog.error("no appid can use")
return False
host = ssl_sock.appid + ".appspot.com"
ssl_sock.host = host
else:
host = ssl_sock.host
# public appid don't keep alive, for quota limit.
if ssl_sock.appid.startswith(
"xxnet-") and ssl_sock.appid[7:].isdigit():
#logging.info("public appid don't keep alive")
#self.keep_alive = 0
return False
#logging.debug("head request %s", host)
request_data = 'HEAD /_gh/ HTTP/1.1\r\nHost: %s\r\n\r\n' % host
response = None
try:
ssl_sock.settimeout(10)
ssl_sock.sock.settimeout(10)
data = request_data.encode()
ret = ssl_sock.send(data)
if ret != len(data):
xlog.warn("head send len:%d %d", ret, len(data))
response = httplib.HTTPResponse(ssl_sock, buffering=True)
response.begin()
status = response.status
if status != 200:
xlog.debug("app head fail status:%d", status)
raise Exception("app check fail")
return True
except httplib.BadStatusLine as e:
inactive_time = time.time() - ssl_sock.last_use_time
xlog.debug("%s keep alive fail, time:%d", ssl_sock.ip,
inactive_time)
return False
except Exception as e:
xlog.debug("head request fail:%r", e)
return False
finally:
if response:
response.close()
def test_gae_ip(ip):
try:
ssl_sock = connect_ssl(ip,
timeout=max_timeout,
openssl_context=openssl_context)
get_ssl_cert_domain(ssl_sock)
appid = appid_manager.get_appid()
check_appid(ssl_sock, appid)
return ssl_sock
except Exception as e:
#xlog.exception("test_gae_ip %s e:%r",ip, e)
return False
def head_request(self, ssl_sock):
if ssl_sock.host == '':
ssl_sock.appid = appid_manager.get_appid()
if not ssl_sock.appid:
xlog.error("no appid can use")
return False
host = ssl_sock.appid + ".appspot.com"
ssl_sock.host = host
else:
host = ssl_sock.host
# public appid don't keep alive, for quota limit.
if ssl_sock.appid.startswith("xxnet-") and ssl_sock.appid[7:].isdigit():
#logging.info("public appid don't keep alive")
#self.keep_alive = 0
return False
#logging.debug("head request %s", host)
request_data = 'HEAD /_gh/ HTTP/1.1\r\nHost: %s\r\n\r\n' % host
response = None
try:
ssl_sock.settimeout(10)
ssl_sock._sock.settimeout(10)
data = request_data.encode()
ret = ssl_sock.send(data)
if ret != len(data):
xlog.warn("head send len:%d %d", ret, len(data))
response = httplib.HTTPResponse(ssl_sock, buffering=True)
response.begin()
status = response.status
if status != 200:
xlog.debug("app head fail status:%d", status)
raise Exception("app check fail %r" % status)
return True
except httplib.BadStatusLine as e:
inactive_time = time.time() - ssl_sock.last_use_time
xlog.debug("%s keep alive fail, time:%d", ssl_sock.ip, inactive_time)
return False
except Exception as e:
xlog.warn("%s head %s request fail:%r", ssl_sock.ip, ssl_sock.appid, e)
return False
finally:
if response:
response.close()
def test_gae_ip(ip, appid=None):
try:
ssl_sock = connect_ssl(ip, timeout=max_timeout, openssl_context=openssl_context)
get_ssl_cert_domain(ssl_sock)
if not appid:
appid = appid_manager.get_appid()
if appid.startswith("xxnet-"):
appid = "xxnet-check"
if not check_appid(ssl_sock, appid, ip):
return False
return ssl_sock
except Exception as e:
#xlog.exception("test_gae_ip %s e:%r",ip, e)
return False
def on_ssl_created_cb(self, ssl_sock):
appid = appid_manager.get_appid()
if not appid:
time.sleep(60)
ssl_sock.close()
raise GAE_Exception(1, "no appid can use")
ssl_sock.appid = appid
ssl_sock.host = ssl_sock.appid + ".appspot.com"
if ssl_sock.h2:
worker = HTTP2_worker(ssl_sock, self.close_cb, self.retry_task_cb)
self.h2_num += 1
else:
worker = HTTP1_worker(ssl_sock, self.close_cb, self.retry_task_cb)
self.h1_num += 1
self.workers.append(worker)
return worker
def on_ssl_created_cb(self, ssl_sock):
appid = appid_manager.get_appid()
if not appid:
time.sleep(60)
ssl_sock.close()
raise GAE_Exception(1, "no appid can use")
ssl_sock.appid = appid
ssl_sock.host = ssl_sock.appid + ".appspot.com"
if ssl_sock.h2:
worker = HTTP2_worker(ssl_sock, self.close_cb, self.retry_task_cb)
self.h2_num += 1
else:
worker = HTTP1_worker(ssl_sock, self.close_cb, self.retry_task_cb)
self.h1_num += 1
self.workers.append(worker)
return worker
def test_app_head(ssl_sock, ip):
appid = appid_manager.get_appid()
request_data = 'HEAD /_gh/ HTTP/1.1\r\nHost: %s.appspot.com\r\n\r\n' % appid
time_start = time.time()
ssl_sock.send(request_data.encode())
response = httplib.HTTPResponse(ssl_sock, buffering=True)
try:
response.begin()
status = response.status
if status != 200:
xlog.debug("app check %s status:%d", ip, status)
raise Exception("app check fail")
except Exception as e:
xlog.exception("test_app_head except:%r", e)
return False
finally:
response.close()
time_stop = time.time()
time_cost = (time_stop - time_start) * 1000
xlog.debug("app check time:%d", time_cost)
return True
def head_request(ssl_sock):
if ssl_sock.host == '':
ssl_sock.appid = appid_manager.get_appid()
if not ssl_sock.appid:
logging.error("no appid can use")
return False
host = ssl_sock.appid + ".appspot.com"
ssl_sock.host = host
else:
host = ssl_sock.host
#logging.debug("head request %s", host)
request_data = 'HEAD /_gh/ HTTP/1.1\r\nHost: %s\r\n\r\n' % host
try:
ssl_sock.settimeout(2)
ssl_sock.send(request_data.encode())
response = httplib.HTTPResponse(ssl_sock, buffering=True)
response.begin()
status = response.status
if status != 200:
logging.debug("app head fail status:%d", status)
raise Exception("app check fail")
content = response.read()
return True
except httplib.BadStatusLine as e:
return False
except Exception as e:
logging.debug("head request fail:%r", e)
return False
finally:
response.close()
def request(headers={}, payload=None):
max_retry = 3
for i in range(max_retry):
ssl_sock = None
try:
ssl_sock = https_manager.get_ssl_connection()
if not ssl_sock:
xlog.debug('create_ssl_connection fail')
continue
if ssl_sock.host == '':
ssl_sock.appid = appid_manager.get_appid()
if not ssl_sock.appid:
google_ip.report_connect_closed(ssl_sock.ip, "no appid")
time.sleep(60)
raise GAE_Exception(1, "no appid can use")
headers['Host'] = ssl_sock.appid + ".appspot.com"
ssl_sock.host = headers['Host']
else:
headers['Host'] = ssl_sock.host
response = _request(ssl_sock, headers, payload)
if not response:
google_ip.report_connect_closed(ssl_sock.ip, "request_fail")
ssl_sock.close()
continue
response.ssl_sock = ssl_sock
return response
except Exception as e:
xlog.exception('request failed:%s', e)
if ssl_sock:
google_ip.report_connect_closed(ssl_sock.ip, "request_except")
ssl_sock.close()
raise GAE_Exception(2, "try max times")
def request(headers={}, payload=None):
max_retry = 3
for i in range(max_retry):
ssl_sock = None
try:
ssl_sock = https_manager.get_ssl_connection()
if not ssl_sock:
xlog.debug('create_ssl_connection fail')
continue
if ssl_sock.host == '':
ssl_sock.appid = appid_manager.get_appid()
if not ssl_sock.appid:
google_ip.report_connect_closed(ssl_sock.ip, "no appid")
time.sleep(60)
raise GAE_Exception(1, "no appid can use")
headers['Host'] = ssl_sock.appid + ".appspot.com"
ssl_sock.host = headers['Host']
else:
headers['Host'] = ssl_sock.host
response = _request(ssl_sock, headers, payload)
if not response:
google_ip.report_connect_closed(ssl_sock.ip, "request_fail")
ssl_sock.close()
continue
response.ssl_sock = ssl_sock
return response
except Exception as e:
xlog.exception('request failed:%s', e)
if ssl_sock:
google_ip.report_connect_closed(ssl_sock.ip, "request_except")
ssl_sock.close()
raise GAE_Exception(2, "try max times")
def __fetchlet(self, range_queue, data_queue, range_delay_size):
headers = dict((k.title(), v) for k, v in self.headers.items())
headers['Connection'] = 'close'
while not self._stopped:
try:
try:
start, end, response = range_queue.get(timeout=1)
if self.expect_begin < start and data_queue.qsize() * self.bufsize + range_delay_size > 30*1024*1024:
range_queue.put((start, end, response))
time.sleep(10)
continue
headers['Range'] = 'bytes=%d-%d' % (start, end)
if not response:
response = fetch(self.method, self.url, headers, self.body)
except Queue.Empty:
continue
except Exception as e:
xlog.warning("RangeFetch fetch response %r in __fetchlet", e)
range_queue.put((start, end, None))
continue
if not response:
xlog.warning('RangeFetch %s return %r', headers['Range'], response)
range_queue.put((start, end, None))
continue
if response.app_status != 200:
xlog.warning('Range Fetch return %s "%s %s" %s ', response.app_status, self.method, self.url, headers['Range'])
if response.app_status == 404:
xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid)
appid_manager.report_not_exist(response.ssl_sock.appid, response.ssl_sock.ip)
appid = appid_manager.get_appid()
if not appid:
xlog.error("no appid left")
self._stopped = True
response.close()
return
if response.app_status == 503:
xlog.warning('APPID %r out of Quota, remove it temporary.', response.ssl_sock.appid)
appid_manager.report_out_of_quota(response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
xlog.error("no appid left")
self._stopped = True
response.close()
return
google_ip.report_connect_closed(response.ssl_sock.ip, "app err")
response.close()
range_queue.put((start, end, None))
continue
if response.getheader('Location'):
self.url = urlparse.urljoin(self.url, response.getheader('Location'))
xlog.info('RangeFetch Redirect(%r)', self.url)
google_ip.report_connect_closed(response.ssl_sock.ip, "reLocation")
response.close()
range_queue.put((start, end, None))
continue
if 200 <= response.status < 300:
content_range = response.getheader('Content-Range')
if not content_range:
xlog.warning('RangeFetch "%s %s" return Content-Range=%r: response headers=%r, retry %s-%s',
self.method, self.url, content_range, response.getheaders(), start, end)
google_ip.report_connect_closed(response.ssl_sock.ip, "no range")
response.close()
range_queue.put((start, end, None))
continue
content_length = int(response.getheader('Content-Length', 0))
xlog.info('>>>>>>>>>>>>>>> [thread %s] %s %s', threading.currentThread().ident, content_length, content_range)
time_last_read = time.time()
while start < end + 1:
try:
data = response.read(self.bufsize)
if not data:
if time.time() - time_last_read > 20:
break
else:
time.sleep(0.1)
continue
time_last_read = time.time()
data_len = len(data)
data_queue.put((start, data))
start += data_len
except Exception as e:
xlog.warning('RangeFetch "%s %s" %s failed: %s', self.method, self.url, headers['Range'], e)
break
if start < end + 1:
xlog.warning('RangeFetch "%s %s" retry %s-%s', self.method, self.url, start, end)
google_ip.report_connect_closed(response.ssl_sock.ip, "down err")
response.close()
range_queue.put((start, end, None))
continue
https_manager.save_ssl_connection_for_reuse(response.ssl_sock)
xlog.info('>>>>>>>>>>>>>>> Successfully reached %d bytes.', start - 1)
else:
xlog.error('RangeFetch %r return %s', self.url, response.status)
google_ip.report_connect_closed(response.ssl_sock.ip, "status err")
response.close()
range_queue.put((start, end, None))
continue
except StandardError as e:
xlog.exception('RangeFetch._fetchlet error:%s', e)
raise
def handler(method, url, headers, body, wfile):
time_request = time.time()
headers = clean_empty_header(headers)
errors = []
response = None
while True:
if time.time() - time_request > 30: #time out
return return_fail_message(wfile)
try:
response = fetch(method, url, headers, body)
if response.app_status != 200:
xlog.warn("fetch gae status:%s url:%s", response.app_status, url)
try:
server_type = response.getheader('Server', "")
if "gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type:
xlog.warn("IP:%s not support GAE, server type:%s", response.ssl_sock.ip, server_type)
google_ip.report_connect_fail(response.ssl_sock.ip, force_remove=True)
response.close()
continue
except Exception as e:
errors.append(e)
xlog.warn('gae_handler.handler %r %s , retry...', e, url)
continue
if response.app_status == 404:
#xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid)
appid_manager.report_not_exist(response.ssl_sock.appid, response.ssl_sock.ip)
google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist")
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html('404 No usable Appid Exists', u'没有可用appid了，请配置可用的appid')
send_response(wfile, 404, body=html.encode('utf-8'))
response.close()
return
else:
response.close()
continue
if response.app_status == 403 or response.app_status == 405: #Method not allowed
# google have changed from gws to gvs, need to remove.
xlog.warning('405 Method not allowed. remove %s ', response.ssl_sock.ip)
# some ip can connect, and server type is gws
# but can't use as GAE server
# so we need remove it immediately
google_ip.report_connect_fail(response.ssl_sock.ip, force_remove=True)
response.close()
continue
if response.app_status == 503:
xlog.warning('APPID %r out of Quota, remove it. %s', response.ssl_sock.appid, response.ssl_sock.ip)
appid_manager.report_out_of_quota(response.ssl_sock.appid)
google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota")
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html('503 No usable Appid Exists', u'appid流量不足，请增加appid')
send_response(wfile, 503, body=html.encode('utf-8'))
response.close()
return
else:
response.close()
continue
if response.app_status < 500:
break
except GAE_Exception as e:
errors.append(e)
xlog.warn("gae_exception:%r %s", e, url)
except Exception as e:
errors.append(e)
xlog.exception('gae_handler.handler %r %s , retry...', e, url)
if response.status == 206:
return RangeFetch(method, url, headers, body, response, wfile).fetch()
try:
wfile.write("HTTP/1.1 %d %s\r\n" % (response.status, response.reason))
response_headers = {}
for key, value in response.getheaders():
key = key.title()
if key == 'Transfer-Encoding':
#http://en.wikipedia.org/wiki/Chunked_transfer_encoding
continue
if key in skip_headers:
continue
response_headers[key] = value
if 'X-Head-Content-Length' in response_headers:
if method == "HEAD":
response_headers['Content-Length'] = response_headers['X-Head-Content-Length']
del response_headers['X-Head-Content-Length']
send_to_browser = True
try:
for key in response_headers:
value = response_headers[key]
send_header(wfile, key, value)
#logging.debug("Head- %s: %s", key, value)
wfile.write("\r\n")
except Exception as e:
send_to_browser = False
xlog.warn("gae_handler.handler send response fail. t:%d e:%r %s", time.time()-time_request, e, url)
if len(response.app_msg):
xlog.warn("APPID error:%d url:%s", response.status, url)
wfile.write(response.app_msg)
google_ip.report_connect_closed(response.ssl_sock.ip, "app err")
response.close()
return
content_length = int(response.getheader('Content-Length', 0))
content_range = response.getheader('Content-Range', '')
if content_range:
start, end, length = tuple(int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)', content_range).group(1, 2, 3))
else:
start, end, length = 0, content_length-1, content_length
body_length = end - start + 1
last_read_time = time.time()
time_response = time.time()
while True:
if start > end:
time_finished = time.time()
if body_length > 1024 and time_finished - time_response > 0:
speed = body_length / (time_finished - time_response)
xlog.info("GAE %d|%s|%d t:%d s:%d hs:%d Spd:%d %d %s",
response.ssl_sock.fd, response.ssl_sock.ip, response.ssl_sock.received_size, (time_finished-time_request)*1000,
length, response.ssl_sock.handshake_time, int(speed), response.status, url)
else:
xlog.info("GAE %d|%s|%d t:%d s:%d hs:%d %d %s",
response.ssl_sock.fd, response.ssl_sock.ip, response.ssl_sock.received_size, (time_finished-time_request)*1000,
length, response.ssl_sock.handshake_time, response.status, url)
response.ssl_sock.received_size += body_length
https_manager.save_ssl_connection_for_reuse(response.ssl_sock, call_time=time_request)
return
data = response.read(config.AUTORANGE_BUFSIZE)
if not data:
if time.time() - last_read_time > 20:
google_ip.report_connect_closed(response.ssl_sock.ip, "down fail")
response.close()
xlog.warn("read timeout t:%d len:%d left:%d %s", (time.time()-time_request)*1000, length, (end-start), url)
return
else:
time.sleep(0.1)
continue
last_read_time = time.time()
data_len = len(data)
start += data_len
if send_to_browser:
try:
ret = wfile.write(data)
if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
xlog.debug("send to browser wfile.write ret:%d", ret)
ret = wfile.write(data)
except Exception as e_b:
if e_b[0] in (errno.ECONNABORTED, errno.EPIPE, errno.ECONNRESET) or 'bad write retry' in repr(e_b):
xlog.warn('gae_handler send to browser return %r %r', e_b, url)
else:
xlog.warn('gae_handler send to browser return %r %r', e_b, url)
send_to_browser = False
except NetWorkIOError as e:
time_except = time.time()
time_cost = time_except - time_request
if e[0] in (errno.ECONNABORTED, errno.EPIPE) or 'bad write retry' in repr(e):
xlog.warn("gae_handler err:%r time:%d %s ", e, time_cost, url)
google_ip.report_connect_closed(response.ssl_sock.ip, "Net")
else:
xlog.exception("gae_handler except:%r %s", e, url)
except Exception as e:
xlog.exception("gae_handler except:%r %s", e, url)
def handler(method, url, headers, body, wfile):
time_request = time.time()
errors = []
response = None
for i in xrange(max_retry):
try:
response = fetch(method, url, headers, body)
if response.app_status != 200:
logging.debug("fetch gae status:%d url:%s", response.app_status, url)
if response.app_status == 404:
logging.warning('APPID %r not exists, remove it.', response.ssl_sock.appid)
appid_manager.report_not_exist(response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html('404 No usable Appid Exists', u'没有可用appid了，请配置可用的appid')
send_response(wfile, 404, body=html.encode('utf-8'))
response.close()
return
else:
response.close()
continue
if response.app_status == 503:
logging.warning('APPID %r out of Quota, remove it.', response.ssl_sock.appid)
appid_manager.report_out_of_quota(response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html('503 No usable Appid Exists', u'appid流量不足，请增加appid')
send_response(wfile, 503, body=html.encode('utf-8'))
response.close()
return
else:
response.close()
continue
if response.app_status < 500:
break
except GAE_Exception as e:
errors.append(e)
logging.warn("gae_exception:%s %r", e, url)
except Exception as e:
errors.append(e)
logging.exception('gae_handler.handler %r %s , retry...', e, url)
if len(errors) == max_retry:
if response and response.app_status >= 500:
status = response.app_status
headers = dict(response.getheaders())
content = response.read()
else:
status = 502
headers = {'Content-Type': 'text/html'}
content = generate_message_html('502 URLFetch failed', 'Local URLFetch %r failed' % url, '<br>'.join(repr(x) for x in errors))
if response:
response.close()
send_response(wfile, status, headers, content.encode('utf-8'))
logging.warn("GAE %d %s %s", status, method, url)
return
if response.status == 206:
return RangeFetch(method, url, headers, body, response, wfile).fetch()
try:
wfile.write("HTTP/1.1 %d %s\r\n" % (response.status, response.reason))
response_headers = {}
for key, value in response.getheaders():
key = key.title()
if key == 'Transfer-Encoding':
continue
if key in skip_headers:
continue
response_headers[key] = value
if method == "HEAD":
if 'X-Head-Content-Length' in response_headers:
response_headers['Content-Length'] = response_headers['X-Head-Content-Length']
del response_headers['X-Head-Content-Length']
for key in response_headers:
value = response_headers[key]
send_header(wfile, key, value)
#logging.debug("Head- %s: %s", key, value)
wfile.write("\r\n")
if len(response.app_msg):
logging.warn("APPID error:%d url:%s", response.status, url)
wfile.write(response.app_msg)
response.close()
return
content_length = int(response.getheader('Content-Length', 0))
content_range = response.getheader('Content-Range', '')
if content_range:
start, end, length = tuple(int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)', content_range).group(1, 2, 3))
else:
start, end, length = 0, content_length-1, content_length
time_start = time.time()
send_to_broswer = True
while True:
data = response.read(config.AUTORANGE_BUFSIZE)
if not data and time.time() - time_start > 20:
response.close()
logging.warn("read timeout t:%d len:%d left:%d %s", (time.time()-time_request)*1000, length, (end-start), url)
return
data_len = len(data)
start += data_len
if send_to_broswer:
try:
ret = wfile.write(data)
if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
logging.debug("send to browser wfile.write ret:%d", ret)
ret = wfile.write(data)
except Exception as e_b:
if e_b[0] in (errno.ECONNABORTED, errno.EPIPE, errno.ECONNRESET) or 'bad write retry' in repr(e_b):
logging.warn('gae_handler send to browser return %r %r', e_b, url)
else:
logging.warn('gae_handler send to browser return %r %r', e_b, url)
send_to_broswer = False
if start >= end:
https_manager.save_ssl_connection_for_reuse(response.ssl_sock)
logging.info("GAE t:%d s:%d %d %s", (time.time()-time_request)*1000, length, response.status, url)
return
except NetWorkIOError as e:
if e[0] in (errno.ECONNABORTED, errno.EPIPE) or 'bad write retry' in repr(e):
logging.warn("gae_handler err:%r %s ", e, url)
else:
logging.exception("gae_handler except:%r %s", e, url)
except Exception as e:
logging.exception("gae_handler except:%r %s", e, url)
def do_AGENT(self):
"""GAE http urlfetch"""
request_headers = dict((k.title(), v) for k, v in self.headers.items())
host = request_headers.get('Host', '')
path = self.parsed_url.path
range_in_query = 'range=' in self.parsed_url.query
special_range = (any(x(host) for x in config.AUTORANGE_HOSTS_MATCH) or path.endswith(config.AUTORANGE_ENDSWITH)) and not path.endswith(config.AUTORANGE_NOENDSWITH)
if 'Range' in request_headers:
m = re.search(r'bytes=(\d+)-', request_headers['Range'])
start = int(m.group(1) if m else 0)
request_headers['Range'] = 'bytes=%d-%d' % (start, start+config.AUTORANGE_MAXSIZE-1)
logging.info('autorange range=%r match url=%r', request_headers['Range'], self.path)
elif not range_in_query and special_range:
logging.info('Found [autorange]endswith match url=%r', self.path)
m = re.search(r'bytes=(\d+)-', request_headers.get('Range', ''))
start = int(m.group(1) if m else 0)
request_headers['Range'] = 'bytes=%d-%d' % (start, start+config.AUTORANGE_MAXSIZE-1)
payload = b''
if 'Content-Length' in request_headers:
try:
payload = self.rfile.read(int(request_headers.get('Content-Length', 0)))
except NetWorkIOError as e:
logging.error('handle_method_urlfetch read payload failed:%s', e)
return
response = None
errors = []
headers_sent = False
for retry in range(config.FETCHMAX_LOCAL):
try:
content_length = 0
kwargs = {}
# TODO: test validate = 1
kwargs['validate'] = 0
time_start = time.time()
response = gae_urlfetch(self.command, self.path, request_headers, payload, **kwargs)
time_stop = time.time()
time_cost = int((time_stop - time_start) * 1000)
if not response:
if retry >= config.FETCHMAX_LOCAL-1:
html = generate_message_html('502 URLFetch failed', 'Local URLFetch %r failed' % self.path, str(errors))
self.wfile.write(b'HTTP/1.0 502\r\nContent-Type: text/html\r\n\r\n' + html.encode('utf-8'))
logging.warning('GET no response %s ', self.path)
return
else:
continue
# appid not exists, try remove it from appid
if response.app_status == 404:
logging.warning('APPID %r not exists, remove it.', response.ssl_sock.appid)
appid_manager.report_not_exist(response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html('404 No usable Appid Exists', u'没有可用appid了，请配置可用的appid')
self.wfile.write(b'HTTP/1.0 404\r\nContent-Type: text/html\r\n\r\n' + html.encode('utf-8'))
response.close()
return
else:
continue
# appid over qouta, switch to next appid
if response.app_status == 503:
logging.warning('APPID %r out of Quota, remove it.', response.ssl_sock.appid)
appid_manager.report_out_of_quota(response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html('503 No usable Appid Exists', u'appid流量不足，请增加appid')
self.wfile.write(b'HTTP/1.0 503\r\nContent-Type: text/html\r\n\r\n' + html.encode('utf-8'))
response.close()
return
else:
continue
# 500 is Web server internal err
if response.app_status == 500 and range_in_query and special_range:
logging.warning('500 with range in query') #, need trying another APPID?
response.close()
continue
if response.app_status == 501:
deploy_url = "http://127.0.0.1:8085/?module=goagent&menu=deploy"
message = u'请重新部署服务端: <a href="%s">%s</a>' % (deploy_url, deploy_url)
html = generate_message_html('Please deploy your new server', message)
self.wfile.write(b'HTTP/1.0 501\r\nContent-Type: text/html\r\n\r\n' + html.encode('utf-8'))
logging.warning('501 Please deploy your new server') #, need trying another APPID?
response.close()
return
if response.app_status != 200 and retry == config.FETCHMAX_LOCAL-1:
logging.warn('GAE %s %s status:%s', self.command, self.path, response.status)
self.wfile.write(('HTTP/1.1 %s\r\n%s\r\n' % (response.status, ''.join('%s: %s\r\n' % (k.title(), v) for k, v in response.getheaders() if k.title() != 'Transfer-Encoding'))))
self.wfile.write(response.read())
response.close()
return
# first response, has no retry.
if not headers_sent:
data_size = response.getheader('Content-Length', '0')
download_speed = int(data_size) * 1000 / time_cost
logging.info('"GAE t:%d speed:%d len:%s status:%s %s %s HTTP/1.1"', time_cost, download_speed, response.getheader('Content-Length', '-'), response.status, self.command, self.path)
if response.status == 206:
# 206 means "Partial Content"
rangefetch = RangeFetch(self.wfile, response, self.command, self.path, self.headers, payload, maxsize=config.AUTORANGE_MAXSIZE, bufsize=config.AUTORANGE_BUFSIZE, waitsize=config.AUTORANGE_WAITSIZE, threads=config.AUTORANGE_THREADS)
return rangefetch.fetch()
if response.getheader('Set-Cookie'):
response.msg['Set-Cookie'] = self.normcookie(response.getheader('Set-Cookie'))
if response.getheader('Content-Disposition') and '"' not in response.getheader('Content-Disposition'):
response.msg['Content-Disposition'] = self.normattachment(response.getheader('Content-Disposition'))
headers_data = 'HTTP/1.1 %s\r\n%s\r\n' % (response.status, ''.join('%s: %s\r\n' % (k.title(), v) for k, v in response.getheaders() if k.title() != 'Transfer-Encoding'))
#logging.debug('headers_data=%s', headers_data)
#self.wfile.write(headers_data.encode() if bytes is not str else headers_data)
self.wfile.write(headers_data)
headers_sent = True
content_length = int(response.getheader('Content-Length', 0))
content_range = response.getheader('Content-Range', '')
accept_ranges = response.getheader('Accept-Ranges', 'none')
if content_range:
start, end, length = tuple(int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)', content_range).group(1, 2, 3))
else:
start, end, length = 0, content_length-1, content_length
if content_length == 0:
https_manager.save_ssl_connection_for_reuse(response.ssl_sock)
return
send_to_broswer = True
while True:
data = response.read(8192) #TODO: loop read until timeout or except.
if not data:
response.close()
return
data_len = len(data)
start += data_len
if send_to_broswer:
try:
ret = self.wfile.write(data)
if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
logging.debug("self.wfile.write ret:%d", ret)
ret = self.wfile.write(data)
except Exception as e_b:
logging.exception('GAEProxyHandler.do_METHOD_AGENT send to browser %r return %r', self.path, e_b)
send_to_broswer = False
if start >= end:
https_manager.save_ssl_connection_for_reuse(response.ssl_sock)
return
except Exception as e:
logging.exception('GAEProxyHandler.do_METHOD_AGENT %r return %r', self.path, e)
errors.append(e)
if response:
response.close()
if e.args[0] in (errno.ECONNABORTED, errno.EPIPE, errno.ECONNRESET):
#logging.debug('GAEProxyHandler.do_METHOD_AGENT return %r', e)
pass
elif e.args[0] == errno.ETIMEDOUT or isinstance(e.args[0], str) and 'timed out' in e.args[0]:
if content_length and accept_ranges == 'bytes':
# we can retry range fetch here
logging.warn('GAEProxyHandler.do_METHOD_AGENT timed out, content_length=%r, url=%r, try again', content_length, self.path)
self.headers['Range'] = 'bytes=%d-%d' % (start, end)
elif isinstance(e, NetWorkIOError) and 'bad write retry' in e.args[-1]:
logging.warn('GAEProxyHandler.do_METHOD_AGENT return %r, abort. url=%r ', e, self.path)
return
else:
logging.exception('GAEProxyHandler.do_METHOD_AGENT %r return %r', self.path, e) #IOError(9, 'Bad file descriptor'), int(e.args[0])
def __fetchlet(self, range_queue, data_queue, range_delay_size):
headers = dict((k.title(), v) for k, v in self.headers.items())
headers['Connection'] = 'close'
while not self._stopped:
try:
try:
start, end, response = range_queue.get(timeout=1)
if self.expect_begin < start and data_queue.qsize(
) * self.bufsize + range_delay_size > 30 * 1024 * 1024:
range_queue.put((start, end, response))
time.sleep(10)
continue
headers['Range'] = 'bytes=%d-%d' % (start, end)
if not response:
response = fetch(self.method, self.url, headers,
self.body)
except Queue.Empty:
continue
except Exception as e:
xlog.warning("RangeFetch fetch response %r in __fetchlet",
e)
range_queue.put((start, end, None))
continue
if not response:
xlog.warning('RangeFetch %s return %r', headers['Range'],
response)
range_queue.put((start, end, None))
continue
if response.app_status != 200:
xlog.warning('Range Fetch return %s "%s %s" %s ',
response.app_status, self.method, self.url,
headers['Range'])
if response.app_status == 404:
xlog.warning('APPID %r not exists, remove it.',
response.ssl_sock.appid)
appid_manager.report_not_exist(response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
xlog.error("no appid left")
self._stopped = True
response.close()
return
if response.app_status == 503:
xlog.warning(
'APPID %r out of Quota, remove it temporary.',
response.ssl_sock.appid)
appid_manager.report_out_of_quota(
response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
xlog.error("no appid left")
self._stopped = True
response.close()
return
response.close()
range_queue.put((start, end, None))
continue
if response.getheader('Location'):
self.url = urlparse.urljoin(self.url,
response.getheader('Location'))
xlog.info('RangeFetch Redirect(%r)', self.url)
response.close()
range_queue.put((start, end, None))
continue
if 200 <= response.status < 300:
content_range = response.getheader('Content-Range')
if not content_range:
xlog.warning(
'RangeFetch "%s %s" return Content-Range=%r: response headers=%r, retry %s-%s',
self.method, self.url, content_range,
response.getheaders(), start, end)
response.close()
range_queue.put((start, end, None))
continue
content_length = int(
response.getheader('Content-Length', 0))
xlog.info('>>>>>>>>>>>>>>> [thread %s] %s %s',
threading.currentThread().ident, content_length,
content_range)
time_last_read = time.time()
while start < end + 1:
try:
data = response.read(self.bufsize)
if not data:
if time.time() - time_last_read > 20:
break
else:
time.sleep(0.1)
continue
time_last_read = time.time()
data_len = len(data)
data_queue.put((start, data))
start += data_len
except Exception as e:
xlog.warning('RangeFetch "%s %s" %s failed: %s',
self.method, self.url,
headers['Range'], e)
break
if start < end + 1:
xlog.warning('RangeFetch "%s %s" retry %s-%s',
self.method, self.url, start, end)
response.close()
range_queue.put((start, end, None))
continue
https_manager.save_ssl_connection_for_reuse(
response.ssl_sock)
xlog.info('>>>>>>>>>>>>>>> Successfully reached %d bytes.',
start - 1)
else:
xlog.error('RangeFetch %r return %s', self.url,
response.status)
response.close()
range_queue.put((start, end, None))
continue
except StandardError as e:
xlog.exception('RangeFetch._fetchlet error:%s', e)
raise
def handler(method, url, headers, body, wfile):
time_request = time.time()
headers = clean_empty_header(headers)
errors = []
response = None
while True:
if time.time() - time_request > 30: #time out
return return_fail_message(wfile)
try:
response = fetch(method, url, headers, body)
if response.app_status != 200:
xlog.warn("fetch gae status:%s url:%s", response.app_status,
url)
if response.app_status == 404:
xlog.warning('APPID %r not exists, remove it.',
response.ssl_sock.appid)
appid_manager.report_not_exist(response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html('404 No usable Appid Exists',
u'没有可用appid了，请配置可用的appid')
send_response(wfile, 404, body=html.encode('utf-8'))
response.close()
return
else:
response.close()
continue
if response.app_status == 403 or response.app_status == 405: #Method not allowed
# google have changed from gws to gvs, need to remove.
xlog.warning('405 Method not allowed. remove %s ',
response.ssl_sock.ip)
# some ip can connect, and server type is gws
# but can't use as GAE server
# so we need remove it immediately
google_ip.report_connect_fail(response.ssl_sock.ip,
force_remove=True)
response.close()
continue
if response.app_status == 503:
xlog.warning('APPID %r out of Quota, remove it.',
response.ssl_sock.appid)
appid_manager.report_out_of_quota(response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html('503 No usable Appid Exists',
u'appid流量不足，请增加appid')
send_response(wfile, 503, body=html.encode('utf-8'))
response.close()
return
else:
response.close()
continue
if response.app_status < 500:
break
except GAE_Exception as e:
errors.append(e)
xlog.warn("gae_exception:%r %s", e, url)
except Exception as e:
errors.append(e)
xlog.exception('gae_handler.handler %r %s , retry...', e, url)
if response.status == 206:
return RangeFetch(method, url, headers, body, response, wfile).fetch()
try:
wfile.write("HTTP/1.1 %d %s\r\n" % (response.status, response.reason))
response_headers = {}
for key, value in response.getheaders():
key = key.title()
if key == 'Transfer-Encoding':
#http://en.wikipedia.org/wiki/Chunked_transfer_encoding
continue
if key in skip_headers:
continue
response_headers[key] = value
if 'X-Head-Content-Length' in response_headers:
if method == "HEAD":
response_headers['Content-Length'] = response_headers[
'X-Head-Content-Length']
del response_headers['X-Head-Content-Length']
send_to_browser = True
try:
for key in response_headers:
value = response_headers[key]
send_header(wfile, key, value)
#logging.debug("Head- %s: %s", key, value)
wfile.write("\r\n")
except Exception as e:
send_to_browser = False
xlog.warn("gae_handler.handler send response fail. t:%d e:%r %s",
time.time() - time_request, e, url)
if len(response.app_msg):
xlog.warn("APPID error:%d url:%s", response.status, url)
wfile.write(response.app_msg)
response.close()
return
content_length = int(response.getheader('Content-Length', 0))
content_range = response.getheader('Content-Range', '')
if content_range:
start, end, length = tuple(
int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)',
content_range).group(1, 2, 3))
else:
start, end, length = 0, content_length - 1, content_length
last_read_time = time.time()
while True:
if start > end:
https_manager.save_ssl_connection_for_reuse(response.ssl_sock)
xlog.info("GAE t:%d s:%d %d %s",
(time.time() - time_request) * 1000, length,
response.status, url)
return
data = response.read(config.AUTORANGE_BUFSIZE)
if not data:
if time.time() - last_read_time > 20:
response.close()
xlog.warn("read timeout t:%d len:%d left:%d %s",
(time.time() - time_request) * 1000, length,
(end - start), url)
return
else:
time.sleep(0.1)
continue
last_read_time = time.time()
data_len = len(data)
start += data_len
if send_to_browser:
try:
ret = wfile.write(data)
if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
xlog.debug("send to browser wfile.write ret:%d", ret)
ret = wfile.write(data)
except Exception as e_b:
if e_b[0] in (errno.ECONNABORTED, errno.EPIPE,
errno.ECONNRESET
) or 'bad write retry' in repr(e_b):
xlog.warn('gae_handler send to browser return %r %r',
e_b, url)
else:
xlog.warn('gae_handler send to browser return %r %r',
e_b, url)
send_to_browser = False
except NetWorkIOError as e:
time_except = time.time()
time_cost = time_except - time_request
if e[0] in (errno.ECONNABORTED,
errno.EPIPE) or 'bad write retry' in repr(e):
xlog.warn("gae_handler err:%r time:%d %s ", e, time_cost, url)
else:
xlog.exception("gae_handler except:%r %s", e, url)
except Exception as e:
xlog.exception("gae_handler except:%r %s", e, url)
def handler(method, url, headers, body, wfile):
time_request = time.time()
errors = []
response = None
while True:
if time.time() - time_request > 30: # time out
html = generate_message_html("504 GoAgent Proxy Time out", u"GoAgent代理处理超时，请查看日志！")
send_response(wfile, 504, body=html.encode("utf-8"))
return
try:
response = fetch(method, url, headers, body)
if response.app_status != 200:
logging.debug("fetch gae status:%s url:%s", response.app_status, url)
if response.app_status == 404:
logging.warning("APPID %r not exists, remove it.", response.ssl_sock.appid)
appid_manager.report_not_exist(response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html("404 No usable Appid Exists", u"没有可用appid了，请配置可用的appid")
send_response(wfile, 404, body=html.encode("utf-8"))
response.close()
return
else:
response.close()
continue
if response.app_status == 405: # Method not allowed
logging.warning("405 Method not allowed. remove %s ", response.ssl_sock.ip)
google_ip.report_connect_fail(response.ssl_sock.ip, force_remove=True)
response.close()
continue
if response.app_status == 503:
logging.warning("APPID %r out of Quota, remove it.", response.ssl_sock.appid)
appid_manager.report_out_of_quota(response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html("503 No usable Appid Exists", u"appid流量不足，请增加appid")
send_response(wfile, 503, body=html.encode("utf-8"))
response.close()
return
else:
response.close()
continue
if response.app_status < 500:
break
except GAE_Exception as e:
errors.append(e)
logging.warn("gae_exception:%s %r", e, url)
except Exception as e:
errors.append(e)
logging.exception("gae_handler.handler %r %s , retry...", e, url)
if len(errors) == max_retry:
if response and response.app_status >= 500:
status = response.app_status
headers = dict(response.getheaders())
content = response.read()
else:
status = 502
headers = {"Content-Type": "text/html"}
content = generate_message_html(
"502 URLFetch failed", "Local URLFetch %r failed" % url, "<br>".join(repr(x) for x in errors)
)
if response:
response.close()
send_response(wfile, status, headers, content.encode("utf-8"))
logging.warn("GAE %d %s %s", status, method, url)
return
if response.status == 206:
return RangeFetch(method, url, headers, body, response, wfile).fetch()
try:
wfile.write("HTTP/1.1 %d %s\r\n" % (response.status, response.reason))
response_headers = {}
for key, value in response.getheaders():
key = key.title()
if key == "Transfer-Encoding":
# http://en.wikipedia.org/wiki/Chunked_transfer_encoding
continue
if key in skip_headers:
continue
response_headers[key] = value
if "X-Head-Content-Length" in response_headers:
if method == "HEAD":
response_headers["Content-Length"] = response_headers["X-Head-Content-Length"]
del response_headers["X-Head-Content-Length"]
for key in response_headers:
value = response_headers[key]
send_header(wfile, key, value)
# logging.debug("Head- %s: %s", key, value)
wfile.write("\r\n")
if len(response.app_msg):
logging.warn("APPID error:%d url:%s", response.status, url)
wfile.write(response.app_msg)
response.close()
return
content_length = int(response.getheader("Content-Length", 0))
content_range = response.getheader("Content-Range", "")
if content_range:
start, end, length = tuple(
int(x) for x in re.search(r"bytes (\d+)-(\d+)/(\d+)", content_range).group(1, 2, 3)
)
else:
start, end, length = 0, content_length - 1, content_length
time_start = time.time()
send_to_broswer = True
while True:
data = response.read(config.AUTORANGE_BUFSIZE)
if not data and time.time() - time_start > 20:
response.close()
logging.warn(
"read timeout t:%d len:%d left:%d %s",
(time.time() - time_request) * 1000,
length,
(end - start),
url,
)
return
data_len = len(data)
start += data_len
if send_to_broswer:
try:
ret = wfile.write(data)
if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
logging.debug("send to browser wfile.write ret:%d", ret)
ret = wfile.write(data)
except Exception as e_b:
if e_b[0] in (errno.ECONNABORTED, errno.EPIPE, errno.ECONNRESET) or "bad write retry" in repr(e_b):
logging.warn("gae_handler send to browser return %r %r", e_b, url)
else:
logging.warn("gae_handler send to browser return %r %r", e_b, url)
send_to_broswer = False
if start >= end:
https_manager.save_ssl_connection_for_reuse(response.ssl_sock)
logging.info("GAE t:%d s:%d %d %s", (time.time() - time_request) * 1000, length, response.status, url)
return
except NetWorkIOError as e:
if e[0] in (errno.ECONNABORTED, errno.EPIPE) or "bad write retry" in repr(e):
logging.warn("gae_handler err:%r %s ", e, url)
else:
logging.exception("gae_handler except:%r %s", e, url)
except Exception as e:
logging.exception("gae_handler except:%r %s", e, url)
def do_AGENT(self):
"""GAE http urlfetch"""
request_headers = dict((k.title(), v) for k, v in self.headers.items())
host = request_headers.get('Host', '')
path = self.parsed_url.path
range_in_query = 'range=' in self.parsed_url.query
special_range = (any(x(host) for x in config.AUTORANGE_HOSTS_MATCH)
or path.endswith(config.AUTORANGE_ENDSWITH)
) and not path.endswith(config.AUTORANGE_NOENDSWITH)
if 'Range' in request_headers:
m = re.search(r'bytes=(\d+)-', request_headers['Range'])
start = int(m.group(1) if m else 0)
request_headers['Range'] = 'bytes=%d-%d' % (
start, start + config.AUTORANGE_MAXSIZE - 1)
logging.info('autorange range=%r match url=%r',
request_headers['Range'], self.path)
elif not range_in_query and special_range:
logging.info('Found [autorange]endswith match url=%r', self.path)
m = re.search(r'bytes=(\d+)-', request_headers.get('Range', ''))
start = int(m.group(1) if m else 0)
request_headers['Range'] = 'bytes=%d-%d' % (
start, start + config.AUTORANGE_MAXSIZE - 1)
payload = b''
if 'Content-Length' in request_headers:
try:
payload = self.rfile.read(
int(request_headers.get('Content-Length', 0)))
except NetWorkIOError as e:
logging.error('handle_method_urlfetch read payload failed:%s',
e)
return
response = None
errors = []
headers_sent = False
for retry in range(config.FETCHMAX_LOCAL):
try:
content_length = 0
kwargs = {}
# TODO: test validate = 1
kwargs['validate'] = 0
time_start = time.time()
response = gae_urlfetch(self.command, self.path,
request_headers, payload, **kwargs)
time_stop = time.time()
time_cost = int((time_stop - time_start) * 1000)
if not response:
if retry >= config.FETCHMAX_LOCAL - 1:
html = generate_message_html(
'502 URLFetch failed',
'Local URLFetch %r failed' % self.path,
str(errors))
self.wfile.write(
b'HTTP/1.0 502\r\nContent-Type: text/html\r\n\r\n'
+ html.encode('utf-8'))
logging.warning('GET no response %s ', self.path)
return
else:
continue
# appid not exists, try remove it from appid
if response.app_status == 404:
logging.warning('APPID %r not exists, remove it.',
response.ssl_sock.appid)
appid_manager.report_not_exist(response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html(
'404 No usable Appid Exists',
u'没有可用appid了，请配置可用的appid')
self.wfile.write(
b'HTTP/1.0 404\r\nContent-Type: text/html\r\n\r\n'
+ html.encode('utf-8'))
response.close()
return
else:
continue
# appid over qouta, switch to next appid
if response.app_status == 503:
logging.warning('APPID %r out of Quota, remove it.',
response.ssl_sock.appid)
appid_manager.report_out_of_quota(response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html(
'503 No usable Appid Exists',
u'appid流量不足，请增加appid')
self.wfile.write(
b'HTTP/1.0 503\r\nContent-Type: text/html\r\n\r\n'
+ html.encode('utf-8'))
response.close()
return
else:
continue
# 500 is Web server internal err
if response.app_status == 500 and range_in_query and special_range:
logging.warning('500 with range in query'
) #, need trying another APPID?
response.close()
continue
if response.app_status == 501:
deploy_url = "http://127.0.0.1:8085/?module=goagent&menu=deploy"
message = u'请重新部署服务端: <a href="%s">%s</a>' % (deploy_url,
deploy_url)
html = generate_message_html(
'Please deploy your new server', message)
self.wfile.write(
b'HTTP/1.0 501\r\nContent-Type: text/html\r\n\r\n' +
html.encode('utf-8'))
logging.warning('501 Please deploy your new server'
) #, need trying another APPID?
response.close()
return
if response.app_status != 200 and retry == config.FETCHMAX_LOCAL - 1:
logging.warn('GAE %s %s status:%s', self.command,
self.path, response.status)
self.wfile.write(
('HTTP/1.1 %s\r\n%s\r\n' % (response.status, ''.join(
'%s: %s\r\n' % (k.title(), v)
for k, v in response.getheaders()
if k.title() != 'Transfer-Encoding'))))
self.wfile.write(response.read())
response.close()
return
# first response, has no retry.
if not headers_sent:
data_size = response.getheader('Content-Length', '0')
download_speed = int(data_size) * 1000 / time_cost
logging.info(
'"GAE t:%d speed:%d len:%s status:%s %s %s HTTP/1.1"',
time_cost, download_speed,
response.getheader('Content-Length', '-'),
response.status, self.command, self.path)
if response.status == 206:
# 206 means "Partial Content"
rangefetch = RangeFetch(
self.wfile,
response,
self.command,
self.path,
self.headers,
payload,
maxsize=config.AUTORANGE_MAXSIZE,
bufsize=config.AUTORANGE_BUFSIZE,
waitsize=config.AUTORANGE_WAITSIZE,
threads=config.AUTORANGE_THREADS)
return rangefetch.fetch()
if response.getheader('Set-Cookie'):
response.msg['Set-Cookie'] = self.normcookie(
response.getheader('Set-Cookie'))
if response.getheader('Content-Disposition'
) and '"' not in response.getheader(
'Content-Disposition'):
response.msg[
'Content-Disposition'] = self.normattachment(
response.getheader('Content-Disposition'))
headers_data = 'HTTP/1.1 %s\r\n%s\r\n' % (
response.status, ''.join(
'%s: %s\r\n' % (k.title(), v)
for k, v in response.getheaders()
if k.title() != 'Transfer-Encoding'))
#logging.debug('headers_data=%s', headers_data)
#self.wfile.write(headers_data.encode() if bytes is not str else headers_data)
self.wfile.write(headers_data)
headers_sent = True
content_length = int(response.getheader('Content-Length', 0))
content_range = response.getheader('Content-Range', '')
accept_ranges = response.getheader('Accept-Ranges', 'none')
if content_range:
start, end, length = tuple(
int(x)
for x in re.search(r'bytes (\d+)-(\d+)/(\d+)',
content_range).group(1, 2, 3))
else:
start, end, length = 0, content_length - 1, content_length
if content_length == 0:
https_manager.save_ssl_connection_for_reuse(
response.ssl_sock)
return
send_to_broswer = True
while True:
data = response.read(
8192) #TODO: loop read until timeout or except.
if not data:
response.close()
return
data_len = len(data)
start += data_len
if send_to_broswer:
try:
ret = self.wfile.write(data)
if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
logging.debug("self.wfile.write ret:%d", ret)
ret = self.wfile.write(data)
except Exception as e_b:
logging.exception(
'GAEProxyHandler.do_METHOD_AGENT send to browser %r return %r',
self.path, e_b)
send_to_broswer = False
if start >= end:
https_manager.save_ssl_connection_for_reuse(
response.ssl_sock)
return
except Exception as e:
logging.exception(
'GAEProxyHandler.do_METHOD_AGENT %r return %r', self.path,
e)
errors.append(e)
if response:
response.close()
if e.args[0] in (errno.ECONNABORTED, errno.EPIPE,
errno.ECONNRESET):
#logging.debug('GAEProxyHandler.do_METHOD_AGENT return %r', e)
pass
elif e.args[0] == errno.ETIMEDOUT or isinstance(
e.args[0], str) and 'timed out' in e.args[0]:
if content_length and accept_ranges == 'bytes':
# we can retry range fetch here
logging.warn(
'GAEProxyHandler.do_METHOD_AGENT timed out, content_length=%r, url=%r, try again',
content_length, self.path)
self.headers['Range'] = 'bytes=%d-%d' % (start, end)
elif isinstance(
e, NetWorkIOError) and 'bad write retry' in e.args[-1]:
logging.warn(
'GAEProxyHandler.do_METHOD_AGENT return %r, abort. url=%r ',
e, self.path)
return
else:
logging.exception(
'GAEProxyHandler.do_METHOD_AGENT %r return %r',
self.path,
e) #IOError(9, 'Bad file descriptor'), int(e.args[0])
def handler(method, url, headers, body, wfile):
time_request = time.time()
errors = []
response = None
while True:
if time.time() - time_request > 90: #time out
return return_fail_message(wfile)
try:
response = fetch(method, url, headers, body)
if response.app_status != 200:
logging.warn("fetch gae status:%s url:%s", response.app_status, url)
if response.app_status == 404:
logging.warning('APPID %r not exists, remove it.', response.ssl_sock.appid)
appid_manager.report_not_exist(response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html('404 No usable Appid Exists', u'没有可用appid了，请配置可用的appid')
send_response(wfile, 404, body=html.encode('utf-8'))
response.close()
return
else:
response.close()
continue
if response.app_status == 403 or response.app_status == 405: #Method not allowed
# google have changed from gws to gvs, need to remove.
logging.warning('405 Method not allowed. remove %s ', response.ssl_sock.ip)
google_ip.report_connect_fail(response.ssl_sock.ip, force_remove=True)
response.close()
continue
if response.app_status == 503:
logging.warning('APPID %r out of Quota, remove it.', response.ssl_sock.appid)
appid_manager.report_out_of_quota(response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html('503 No usable Appid Exists', u'appid流量不足，请增加appid')
send_response(wfile, 503, body=html.encode('utf-8'))
response.close()
return
else:
response.close()
continue
if response.app_status < 500:
break
except GAE_Exception as e:
errors.append(e)
logging.warn("gae_exception:%r %s", e, url)
except Exception as e:
errors.append(e)
logging.exception('gae_handler.handler %r %s , retry...', e, url)
if response.status == 206:
return RangeFetch(method, url, headers, body, response, wfile).fetch()
try:
wfile.write("HTTP/1.1 %d %s\r\n" % (response.status, response.reason))
response_headers = {}
for key, value in response.getheaders():
key = key.title()
if key == 'Transfer-Encoding':
#http://en.wikipedia.org/wiki/Chunked_transfer_encoding
continue
if key in skip_headers:
continue
response_headers[key] = value
if 'X-Head-Content-Length' in response_headers:
if method == "HEAD":
response_headers['Content-Length'] = response_headers['X-Head-Content-Length']
del response_headers['X-Head-Content-Length']
for key in response_headers:
value = response_headers[key]
send_header(wfile, key, value)
#logging.debug("Head- %s: %s", key, value)
wfile.write("\r\n")
if len(response.app_msg):
logging.warn("APPID error:%d url:%s", response.status, url)
wfile.write(response.app_msg)
response.close()
return
content_length = int(response.getheader('Content-Length', 0))
content_range = response.getheader('Content-Range', '')
if content_range:
start, end, length = tuple(int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)', content_range).group(1, 2, 3))
else:
start, end, length = 0, content_length-1, content_length
send_to_broswer = True
while True:
time_start = time.time()
data = response.read(config.AUTORANGE_BUFSIZE)
if not data and time.time() - time_start > 20:
response.close()
logging.warn("read timeout t:%d len:%d left:%d %s", (time.time()-time_request)*1000, length, (end-start), url)
return
data_len = len(data)
start += data_len
if send_to_broswer:
try:
ret = wfile.write(data)
if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
logging.debug("send to browser wfile.write ret:%d", ret)
ret = wfile.write(data)
except Exception as e_b:
if e_b[0] in (errno.ECONNABORTED, errno.EPIPE, errno.ECONNRESET) or 'bad write retry' in repr(e_b):
logging.warn('gae_handler send to browser return %r %r', e_b, url)
else:
logging.warn('gae_handler send to browser return %r %r', e_b, url)
send_to_broswer = False
if start >= end:
https_manager.save_ssl_connection_for_reuse(response.ssl_sock)
logging.info("GAE t:%d s:%d %d %s", (time.time()-time_request)*1000, length, response.status, url)
return
except NetWorkIOError as e:
time_except = time.time()
time_cost = time_except - time_request
if e[0] in (errno.ECONNABORTED, errno.EPIPE) or 'bad write retry' in repr(e):
logging.warn("gae_handler err:%r time:%d %s ", e, time_cost, url)
else:
logging.exception("gae_handler except:%r %s", e, url)
except Exception as e:
logging.exception("gae_handler except:%r %s", e, url)
def handler(method, url, headers, body, wfile):
time_request = time.time()
errors = []
response = None
for i in xrange(max_retry):
try:
response = fetch(method, url, headers, body)
if response.app_status != 200:
logging.debug("fetch gae status:%d url:%s",
response.app_status, url)
if response.app_status == 404:
logging.warning('APPID %r not exists, remove it.',
response.ssl_sock.appid)
appid_manager.report_not_exist(response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html('404 No usable Appid Exists',
u'没有可用appid了，请配置可用的appid')
send_response(wfile, 404, body=html.encode('utf-8'))
response.close()
return
else:
response.close()
continue
if response.app_status == 503:
logging.warning('APPID %r out of Quota, remove it.',
response.ssl_sock.appid)
appid_manager.report_out_of_quota(response.ssl_sock.appid)
appid = appid_manager.get_appid()
if not appid:
html = generate_message_html('503 No usable Appid Exists',
u'appid流量不足，请增加appid')
send_response(wfile, 503, body=html.encode('utf-8'))
response.close()
return
else:
response.close()
continue
if response.app_status < 500:
break
except GAE_Exception as e:
errors.append(e)
logging.warn("gae_exception:%s %r", e, url)
except Exception as e:
errors.append(e)
logging.exception('gae_handler.handler %r %s , retry...', e, url)
if len(errors) == max_retry:
if response and response.app_status >= 500:
status = response.app_status
headers = dict(response.getheaders())
content = response.read()
else:
status = 502
headers = {'Content-Type': 'text/html'}
content = generate_message_html(
'502 URLFetch failed', 'Local URLFetch %r failed' % url,
'<br>'.join(repr(x) for x in errors))
if response:
response.close()
send_response(wfile, status, headers, content.encode('utf-8'))
logging.warn("GAE %d %s %s", status, method, url)
return
if response.status == 206:
return RangeFetch(method, url, headers, body, response, wfile).fetch()
try:
wfile.write("HTTP/1.1 %d\r\n" % response.status)
response_headers = {}
for key, value in response.getheaders():
key = key.title()
if key == 'Transfer-Encoding':
continue
if key in skip_headers:
continue
response_headers[key] = value
if method == "HEAD":
if 'X-Head-Content-Length' in response_headers:
response_headers['Content-Length'] = response_headers[
'X-Head-Content-Length']
del response_headers['X-Head-Content-Length']
for key in response_headers:
value = response_headers[key]
send_header(wfile, key, value)
#logging.debug("Head- %s: %s", key, value)
wfile.write("\r\n")
if len(response.app_msg):
logging.warn("APPID error:%d url:%s", response.status, url)
wfile.write(response.app_msg)
response.close()
return
content_length = int(response.getheader('Content-Length', 0))
content_range = response.getheader('Content-Range', '')
if content_range:
start, end, length = tuple(
int(x) for x in re.search(r'bytes (\d+)-(\d+)/(\d+)',
content_range).group(1, 2, 3))
else:
start, end, length = 0, content_length - 1, content_length
time_start = time.time()
send_to_broswer = True
while True:
data = response.read(config.AUTORANGE_BUFSIZE)
if not data and time.time() - time_start > 20:
response.close()
logging.warn("read timeout t:%d len:%d left:%d %s",
(time.time() - time_request) * 1000, length,
(end - start), url)
return
data_len = len(data)
start += data_len
if send_to_broswer:
try:
ret = wfile.write(data)
if ret == ssl.SSL_ERROR_WANT_WRITE or ret == ssl.SSL_ERROR_WANT_READ:
logging.debug("send to browser wfile.write ret:%d",
ret)
ret = wfile.write(data)
except Exception as e_b:
if e_b[0] in (errno.ECONNABORTED, errno.EPIPE,
errno.ECONNRESET
) or 'bad write retry' in repr(e_b):
logging.warn(
'gae_handler send to browser return %r %r', e_b,
url)
else:
logging.warn(
'gae_handler send to browser return %r %r', e_b,
url)
send_to_broswer = False
if start >= end:
https_manager.save_ssl_connection_for_reuse(response.ssl_sock)
logging.info("GAE t:%d s:%d %d %s",
(time.time() - time_request) * 1000, length,
response.status, url)
return
except NetWorkIOError as e:
if e[0] in (errno.ECONNABORTED,
errno.EPIPE) or 'bad write retry' in repr(e):
logging.warn("gae_handler err:%r %s ", e, url)
else:
logging.exception("gae_handler except:%r %s", e, url)
except Exception as e:
logging.exception("gae_handler except:%r %s", e, url)