def unpack_response(response):
try:
data = response.task.read(size=2)
if not data:
raise GAE_Exception(600, "get protocol head fail")
headers_length, = struct.unpack('!h', data)
data = response.task.read(size=headers_length)
if not data:
raise GAE_Exception(
600, "get protocol head fail, len:%d" % headers_length)
raw_response_line, headers_data = inflate(data).split('\r\n', 1)
_, status, reason = raw_response_line.split(None, 2)
response.app_status = int(status)
response.app_reason = reason.strip()
headers_block, app_msg = headers_data.split('\r\n\r\n')
headers_pairs = headers_block.split('\r\n')
response.headers = {}
for pair in headers_pairs:
if not pair:
break
k, v = pair.split(': ', 1)
response.headers[k] = v
response.app_msg = app_msg
return response
except Exception as e:
response.worker.close("unpack protocol error")
google_ip.recheck_ip(response.ssl_sock.ip)
raise GAE_Exception(600, "unpack protocol:%r" % e)
def work_loop(self):
while connect_control.keep_running and self.keep_running:
task = self.task_queue.get(True)
if not task:
# None task means exit
self.accept_task = False
self.keep_running = False
return
elif task == "ping":
if not self.head_request():
# now many gvs don't support gae
self.accept_task = False
self.keep_running = False
if self.task is not None:
self.retry_task_cb(self.task)
self.task = None
google_ip.recheck_ip(self.ssl_sock.ip)
self.close("keep alive")
return
else:
self.last_active_time = time.time()
continue
# xlog.debug("http1 get task")
time_now = time.time()
self.last_request_time = time_now
self.request_task(task)
def work_loop(self):
while connect_control.keep_running and self.keep_running:
task = self.task_queue.get(True)
if not task:
# None task means exit
self.accept_task = False
self.keep_running = False
return
elif task == "ping":
if not self.head_request():
# now many gvs don't support gae
self.accept_task = False
self.keep_running = False
if self.task is not None:
self.retry_task_cb(self.task)
self.task = None
google_ip.recheck_ip(self.ssl_sock.ip)
self.close("keep alive")
return
else:
self.last_active_time = time.time()
continue
xlog.debug("http1 get task")
time_now = time.time()
self.last_request_time = time_now
self.request_task(task)
def unpack_response(response):
try:
data = response.body.get(2)
headers_length, = struct.unpack('!h', data)
data = response.body.get(headers_length)
raw_response_line, headers_data = inflate(data).split('\r\n', 1)
_, status, reason = raw_response_line.split(None, 2)
response.app_status = int(status)
response.app_reason = reason.strip()
headers_block, app_msg = headers_data.split('\r\n\r\n')
headers_pairs = headers_block.split('\r\n')
response.headers = {}
for pair in headers_pairs:
if not pair:
break
k, v = pair.split(': ', 1)
response.headers[k] = v
response.app_msg = app_msg
return response
except Exception as e:
response.worker.close("unpack protocol error")
google_ip.recheck_ip(response.ssl_sock.ip)
raise GAE_Exception("unpack protocol:%r", e)
def work_loop(self):
last_ssl_active_time = self.ssl_sock.create_time
last_request_time = time.time()
while connect_control.keep_running and self.keep_running:
time_to_ping = min(0, 55 - (time.time() - last_ssl_active_time))
try:
task = self.task_queue.get(True, timeout=time_to_ping)
if not task:
# None task to exit
return
except:
if time.time() - last_request_time > self.idle_time:
self.close("idle")
return
last_ssl_active_time = time.time()
if not self.head_request():
# google_ip.report_connect_fail(self.ssl_sock.ip, force_remove=True)
# now many gvs don't support gae
google_ip.recheck_ip(self.ssl_sock.ip)
self.close("keep alive, maybe not support")
return
else:
continue
last_request_time = time.time()
self.request_task(task)
def work_loop(self):
last_ssl_active_time = self.ssl_sock.create_time
last_request_time = time.time()
while connect_control.keep_running and self.keep_running:
time_to_ping0 = 55 - (time.time() - last_ssl_active_time)
time_to_ping = max(0, time_to_ping0)
try:
task = self.task_queue.get(True, timeout=time_to_ping)
if not task:
# None task to exit
return
except:
if time.time() - last_request_time > self.idle_time:
self.close("idle")
return
# public appid don't keep alive, for quota limit.
if self.ssl_sock.appid in config.PUBLIC_APPIDS:
#xlog.info("public appid don't keep alive")
self.close("public appid")
return
last_ssl_active_time = time.time()
if not self.head_request():
# now many gvs don't support gae
google_ip.recheck_ip(self.ssl_sock.ip)
self.close("keep alive")
return
else:
continue
last_request_time = time.time()
self.request_task(task)
def work_loop(self):
last_ssl_active_time = self.ssl_sock.create_time
last_request_time = time.time()
while connect_control.keep_running and self.keep_running:
time_to_ping = min(0, 55 - (time.time() - last_ssl_active_time))
try:
task = self.task_queue.get(True, timeout=time_to_ping)
if not task:
# None task to exit
return
except:
if time.time() - last_request_time > self.idle_time:
self.close("idle")
return
last_ssl_active_time = time.time()
if not self.head_request():
# google_ip.report_connect_fail(self.ssl_sock.ip, force_remove=True)
# now many gvs don't support gae
google_ip.recheck_ip(self.ssl_sock.ip)
self.close("keep alive, maybe not support")
return
else:
continue
last_request_time = time.time()
self.request_task(task)
def request_gae_server(headers, body):
# process on http protocol
# process status code return by http server
# raise error, let up layer retry.
response = http_dispatch.request(headers, body)
if not response:
raise GAE_Exception(600, "fetch gae fail")
if response.status >= 600:
raise GAE_Exception(response.status,
"fetch gae fail:%d" % response.status)
server_type = response.headers.get("server", "")
content_type = response.headers.get("content-type", "")
if ("gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type) or \
response.status == 403 or response.status == 405:
# some ip can connect, and server type can be gws
# but can't use as GAE server
# so we need remove it immediately
xlog.warn("IP:%s not support GAE, server:%s status:%d",
response.ssl_sock.ip, server_type, response.status)
google_ip.recheck_ip(response.ssl_sock.ip)
response.worker.close("ip not support GAE")
raise GAE_Exception(602, "ip not support GAE")
if response.status == 404:
# xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid)
appid_manager.report_not_exist(response.ssl_sock.appid,
response.ssl_sock.ip)
# google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist")
response.worker.close("appid not exist:%s" % response.ssl_sock.appid)
raise GAE_Exception(603, "appid not support GAE")
if response.status == 503:
appid = response.ssl_sock.appid
xlog.warning('APPID %r out of Quota, remove it. %s', appid,
response.ssl_sock.ip)
appid_manager.report_out_of_quota(appid)
# google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota")
response.worker.close("appid out of quota:%s" % appid)
raise GAE_Exception(604, "appid out of quota:%s" % appid)
if response.status > 300:
raise GAE_Exception(605, "status:%d" % response.status)
if response.status != 200:
xlog.warn("GAE %s appid:%s status:%d", response.ssl_sock.ip,
response.ssl_sock.appid, response.status)
else:
xlog.info('good ip num:%d, bad ip num:%s', google_ip.good_ip_num,
google_ip.bad_ip_num)
return response
def request_gae_server(headers, body, url, timeout):
# process on http protocol
# process status code return by http server
# raise error, let up layer retry.
response = http_dispatch.request(headers, body, url, timeout)
if not response:
raise GAE_Exception(600, "fetch gae fail")
if response.status >= 600:
raise GAE_Exception(
response.status, "fetch gae fail:%d" % response.status)
server_type = response.getheader("server", "")
# content_type = response.getheaders("content-type", "")
if ("gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type) or \
response.status == 403 or response.status == 405:
# some ip can connect, and server type can be gws
# but can't use as GAE server
# so we need remove it immediately
xlog.warn("IP:%s not support GAE, headers:%s status:%d", response.ssl_sock.ip, response.headers,
response.status)
google_ip.recheck_ip(response.ssl_sock.ip)
response.worker.close("ip not support GAE")
raise GAE_Exception(602, "ip not support GAE")
if response.status == 404:
# xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid)
appid_manager.report_not_exist(
response.ssl_sock.appid, response.ssl_sock.ip)
# google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist")
response.worker.close("appid not exist:%s" % response.ssl_sock.appid)
raise GAE_Exception(603, "appid not support GAE")
if response.status == 503:
appid = response.ssl_sock.appid
xlog.warning('APPID %r out of Quota, remove it. %s',
appid, response.ssl_sock.ip)
appid_manager.report_out_of_quota(appid)
# google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota")
response.worker.close("appid out of quota:%s" % appid)
raise GAE_Exception(604, "appid out of quota:%s" % appid)
if response.status > 300:
raise GAE_Exception(605, "status:%d" % response.status)
if response.status != 200:
xlog.warn("GAE %s appid:%s status:%d", response.ssl_sock.ip,
response.ssl_sock.appid, response.status)
return response
def request_gae_server(headers, body, url, timeout):
response = http_dispatch.request(headers, body, url, timeout)
if not response:
raise GAE_Exception(600, "fetch gae fail")
if response.status >= 600:
raise GAE_Exception(response.status,
"fetch gae fail:%d" % response.status)
server_type = response.getheader("server", "")
if ("gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type) or \
response.status == 403 or response.status == 405:
xlog.warn("IP:%s not support GAE, headers:%s status:%d",
response.ssl_sock.ip, response.headers, response.status)
google_ip.recheck_ip(response.ssl_sock.ip)
response.worker.close("ip not support GAE")
raise GAE_Exception(602, "ip not support GAE")
if response.status == 404:
# xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid)
appid_manager.report_not_exist(response.ssl_sock.appid,
response.ssl_sock.ip)
# google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist")
response.worker.close("appid not exist:%s" % response.ssl_sock.appid)
raise GAE_Exception(603, "appid not support GAE")
if response.status == 503:
appid = response.ssl_sock.appid
xlog.warning('APPID %r out of Quota, remove it. %s', appid,
response.ssl_sock.ip)
appid_manager.report_out_of_quota(appid)
# google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota")
response.worker.close("appid out of quota:%s" % appid)
raise GAE_Exception(604, "appid out of quota:%s" % appid)
if response.status > 300:
raise GAE_Exception(605, "status:%d" % response.status)
if response.status != 200:
xlog.warn("GAE %s appid:%s status:%d", response.ssl_sock.ip,
response.ssl_sock.appid, response.status)
else:
xlog.info('good ip num:%d, bad ip num:%s', google_ip.good_ip_num,
google_ip.bad_ip_num)
return response
def request_gae_server(headers, body, url, timeout):
response = http_dispatch.request(headers, body, url, timeout)
if not response:
raise GAE_Exception(600, "fetch gae fail")
if response.status >= 600:
raise GAE_Exception(
response.status, "fetch gae fail:%d" % response.status)
server_type = response.getheader("server", "")
if ("gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type) or \
response.status == 403 or response.status == 405:
xlog.warn("IP:%s not support GAE, headers:%s status:%d", response.ssl_sock.ip, response.headers,
response.status)
google_ip.recheck_ip(response.ssl_sock.ip)
response.worker.close("ip not support GAE")
raise GAE_Exception(602, "ip not support GAE")
if response.status == 404:
# xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid)
appid_manager.report_not_exist(
response.ssl_sock.appid, response.ssl_sock.ip)
# google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist")
response.worker.close("appid not exist:%s" % response.ssl_sock.appid)
raise GAE_Exception(603, "appid not support GAE")
if response.status == 503:
appid = response.ssl_sock.appid
xlog.warning('APPID %r out of Quota, remove it. %s',
appid, response.ssl_sock.ip)
appid_manager.report_out_of_quota(appid)
# google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota")
response.worker.close("appid out of quota:%s" % appid)
raise GAE_Exception(604, "appid out of quota:%s" % appid)
if response.status > 300:
raise GAE_Exception(605, "status:%d" % response.status)
if response.status != 200:
xlog.warn("GAE %s appid:%s status:%d", response.ssl_sock.ip, response.ssl_sock.appid, response.status)
else:
xlog.info('good ip num:%d, bad ip num:%s', google_ip.good_ip_num, google_ip.bad_ip_num)
return response
def work_loop(self):
while connect_control.keep_running and self.keep_running:
task = self.task_queue.get(True)
if not task:
# None task to exit
return
elif task == "ping":
self.last_active_time = time.time()
if not self.head_request():
# now many gvs don't support gae
google_ip.recheck_ip(self.ssl_sock.ip)
self.close("keep alive")
return
else:
continue
# xlog.debug("http1 get task")
time_now = time.time()
self.last_request_time = time_now
self.last_active_time = time_now
self.request_task(task)
