def unpack_response(response): try: data = response.task.read(size=2) if not data: raise GAE_Exception(600, "get protocol head fail") headers_length, = struct.unpack('!h', data) data = response.task.read(size=headers_length) if not data: raise GAE_Exception( 600, "get protocol head fail, len:%d" % headers_length) raw_response_line, headers_data = inflate(data).split('\r\n', 1) _, status, reason = raw_response_line.split(None, 2) response.app_status = int(status) response.app_reason = reason.strip() headers_block, app_msg = headers_data.split('\r\n\r\n') headers_pairs = headers_block.split('\r\n') response.headers = {} for pair in headers_pairs: if not pair: break k, v = pair.split(': ', 1) response.headers[k] = v response.app_msg = app_msg return response except Exception as e: response.worker.close("unpack protocol error") google_ip.recheck_ip(response.ssl_sock.ip) raise GAE_Exception(600, "unpack protocol:%r" % e)
def work_loop(self): while connect_control.keep_running and self.keep_running: task = self.task_queue.get(True) if not task: # None task means exit self.accept_task = False self.keep_running = False return elif task == "ping": if not self.head_request(): # now many gvs don't support gae self.accept_task = False self.keep_running = False if self.task is not None: self.retry_task_cb(self.task) self.task = None google_ip.recheck_ip(self.ssl_sock.ip) self.close("keep alive") return else: self.last_active_time = time.time() continue # xlog.debug("http1 get task") time_now = time.time() self.last_request_time = time_now self.request_task(task)
def work_loop(self): while connect_control.keep_running and self.keep_running: task = self.task_queue.get(True) if not task: # None task means exit self.accept_task = False self.keep_running = False return elif task == "ping": if not self.head_request(): # now many gvs don't support gae self.accept_task = False self.keep_running = False if self.task is not None: self.retry_task_cb(self.task) self.task = None google_ip.recheck_ip(self.ssl_sock.ip) self.close("keep alive") return else: self.last_active_time = time.time() continue xlog.debug("http1 get task") time_now = time.time() self.last_request_time = time_now self.request_task(task)
def unpack_response(response): try: data = response.body.get(2) headers_length, = struct.unpack('!h', data) data = response.body.get(headers_length) raw_response_line, headers_data = inflate(data).split('\r\n', 1) _, status, reason = raw_response_line.split(None, 2) response.app_status = int(status) response.app_reason = reason.strip() headers_block, app_msg = headers_data.split('\r\n\r\n') headers_pairs = headers_block.split('\r\n') response.headers = {} for pair in headers_pairs: if not pair: break k, v = pair.split(': ', 1) response.headers[k] = v response.app_msg = app_msg return response except Exception as e: response.worker.close("unpack protocol error") google_ip.recheck_ip(response.ssl_sock.ip) raise GAE_Exception("unpack protocol:%r", e)
def work_loop(self): last_ssl_active_time = self.ssl_sock.create_time last_request_time = time.time() while connect_control.keep_running and self.keep_running: time_to_ping = min(0, 55 - (time.time() - last_ssl_active_time)) try: task = self.task_queue.get(True, timeout=time_to_ping) if not task: # None task to exit return except: if time.time() - last_request_time > self.idle_time: self.close("idle") return last_ssl_active_time = time.time() if not self.head_request(): # google_ip.report_connect_fail(self.ssl_sock.ip, force_remove=True) # now many gvs don't support gae google_ip.recheck_ip(self.ssl_sock.ip) self.close("keep alive, maybe not support") return else: continue last_request_time = time.time() self.request_task(task)
def work_loop(self): last_ssl_active_time = self.ssl_sock.create_time last_request_time = time.time() while connect_control.keep_running and self.keep_running: time_to_ping0 = 55 - (time.time() - last_ssl_active_time) time_to_ping = max(0, time_to_ping0) try: task = self.task_queue.get(True, timeout=time_to_ping) if not task: # None task to exit return except: if time.time() - last_request_time > self.idle_time: self.close("idle") return # public appid don't keep alive, for quota limit. if self.ssl_sock.appid in config.PUBLIC_APPIDS: #xlog.info("public appid don't keep alive") self.close("public appid") return last_ssl_active_time = time.time() if not self.head_request(): # now many gvs don't support gae google_ip.recheck_ip(self.ssl_sock.ip) self.close("keep alive") return else: continue last_request_time = time.time() self.request_task(task)
def request_gae_server(headers, body): # process on http protocol # process status code return by http server # raise error, let up layer retry. response = http_dispatch.request(headers, body) if not response: raise GAE_Exception(600, "fetch gae fail") if response.status >= 600: raise GAE_Exception(response.status, "fetch gae fail:%d" % response.status) server_type = response.headers.get("server", "") content_type = response.headers.get("content-type", "") if ("gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type) or \ response.status == 403 or response.status == 405: # some ip can connect, and server type can be gws # but can't use as GAE server # so we need remove it immediately xlog.warn("IP:%s not support GAE, server:%s status:%d", response.ssl_sock.ip, server_type, response.status) google_ip.recheck_ip(response.ssl_sock.ip) response.worker.close("ip not support GAE") raise GAE_Exception(602, "ip not support GAE") if response.status == 404: # xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid) appid_manager.report_not_exist(response.ssl_sock.appid, response.ssl_sock.ip) # google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist") response.worker.close("appid not exist:%s" % response.ssl_sock.appid) raise GAE_Exception(603, "appid not support GAE") if response.status == 503: appid = response.ssl_sock.appid xlog.warning('APPID %r out of Quota, remove it. %s', appid, response.ssl_sock.ip) appid_manager.report_out_of_quota(appid) # google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota") response.worker.close("appid out of quota:%s" % appid) raise GAE_Exception(604, "appid out of quota:%s" % appid) if response.status > 300: raise GAE_Exception(605, "status:%d" % response.status) if response.status != 200: xlog.warn("GAE %s appid:%s status:%d", response.ssl_sock.ip, response.ssl_sock.appid, response.status) else: xlog.info('good ip num:%d, bad ip num:%s', google_ip.good_ip_num, google_ip.bad_ip_num) return response
def request_gae_server(headers, body, url, timeout): # process on http protocol # process status code return by http server # raise error, let up layer retry. response = http_dispatch.request(headers, body, url, timeout) if not response: raise GAE_Exception(600, "fetch gae fail") if response.status >= 600: raise GAE_Exception( response.status, "fetch gae fail:%d" % response.status) server_type = response.getheader("server", "") # content_type = response.getheaders("content-type", "") if ("gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type) or \ response.status == 403 or response.status == 405: # some ip can connect, and server type can be gws # but can't use as GAE server # so we need remove it immediately xlog.warn("IP:%s not support GAE, headers:%s status:%d", response.ssl_sock.ip, response.headers, response.status) google_ip.recheck_ip(response.ssl_sock.ip) response.worker.close("ip not support GAE") raise GAE_Exception(602, "ip not support GAE") if response.status == 404: # xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid) appid_manager.report_not_exist( response.ssl_sock.appid, response.ssl_sock.ip) # google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist") response.worker.close("appid not exist:%s" % response.ssl_sock.appid) raise GAE_Exception(603, "appid not support GAE") if response.status == 503: appid = response.ssl_sock.appid xlog.warning('APPID %r out of Quota, remove it. %s', appid, response.ssl_sock.ip) appid_manager.report_out_of_quota(appid) # google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota") response.worker.close("appid out of quota:%s" % appid) raise GAE_Exception(604, "appid out of quota:%s" % appid) if response.status > 300: raise GAE_Exception(605, "status:%d" % response.status) if response.status != 200: xlog.warn("GAE %s appid:%s status:%d", response.ssl_sock.ip, response.ssl_sock.appid, response.status) return response
def request_gae_server(headers, body, url, timeout): response = http_dispatch.request(headers, body, url, timeout) if not response: raise GAE_Exception(600, "fetch gae fail") if response.status >= 600: raise GAE_Exception(response.status, "fetch gae fail:%d" % response.status) server_type = response.getheader("server", "") if ("gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type) or \ response.status == 403 or response.status == 405: xlog.warn("IP:%s not support GAE, headers:%s status:%d", response.ssl_sock.ip, response.headers, response.status) google_ip.recheck_ip(response.ssl_sock.ip) response.worker.close("ip not support GAE") raise GAE_Exception(602, "ip not support GAE") if response.status == 404: # xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid) appid_manager.report_not_exist(response.ssl_sock.appid, response.ssl_sock.ip) # google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist") response.worker.close("appid not exist:%s" % response.ssl_sock.appid) raise GAE_Exception(603, "appid not support GAE") if response.status == 503: appid = response.ssl_sock.appid xlog.warning('APPID %r out of Quota, remove it. %s', appid, response.ssl_sock.ip) appid_manager.report_out_of_quota(appid) # google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota") response.worker.close("appid out of quota:%s" % appid) raise GAE_Exception(604, "appid out of quota:%s" % appid) if response.status > 300: raise GAE_Exception(605, "status:%d" % response.status) if response.status != 200: xlog.warn("GAE %s appid:%s status:%d", response.ssl_sock.ip, response.ssl_sock.appid, response.status) else: xlog.info('good ip num:%d, bad ip num:%s', google_ip.good_ip_num, google_ip.bad_ip_num) return response
def request_gae_server(headers, body, url, timeout): response = http_dispatch.request(headers, body, url, timeout) if not response: raise GAE_Exception(600, "fetch gae fail") if response.status >= 600: raise GAE_Exception( response.status, "fetch gae fail:%d" % response.status) server_type = response.getheader("server", "") if ("gws" not in server_type and "Google Frontend" not in server_type and "GFE" not in server_type) or \ response.status == 403 or response.status == 405: xlog.warn("IP:%s not support GAE, headers:%s status:%d", response.ssl_sock.ip, response.headers, response.status) google_ip.recheck_ip(response.ssl_sock.ip) response.worker.close("ip not support GAE") raise GAE_Exception(602, "ip not support GAE") if response.status == 404: # xlog.warning('APPID %r not exists, remove it.', response.ssl_sock.appid) appid_manager.report_not_exist( response.ssl_sock.appid, response.ssl_sock.ip) # google_ip.report_connect_closed(response.ssl_sock.ip, "appid not exist") response.worker.close("appid not exist:%s" % response.ssl_sock.appid) raise GAE_Exception(603, "appid not support GAE") if response.status == 503: appid = response.ssl_sock.appid xlog.warning('APPID %r out of Quota, remove it. %s', appid, response.ssl_sock.ip) appid_manager.report_out_of_quota(appid) # google_ip.report_connect_closed(response.ssl_sock.ip, "out of quota") response.worker.close("appid out of quota:%s" % appid) raise GAE_Exception(604, "appid out of quota:%s" % appid) if response.status > 300: raise GAE_Exception(605, "status:%d" % response.status) if response.status != 200: xlog.warn("GAE %s appid:%s status:%d", response.ssl_sock.ip, response.ssl_sock.appid, response.status) else: xlog.info('good ip num:%d, bad ip num:%s', google_ip.good_ip_num, google_ip.bad_ip_num) return response
def work_loop(self): while connect_control.keep_running and self.keep_running: task = self.task_queue.get(True) if not task: # None task to exit return elif task == "ping": self.last_active_time = time.time() if not self.head_request(): # now many gvs don't support gae google_ip.recheck_ip(self.ssl_sock.ip) self.close("keep alive") return else: continue # xlog.debug("http1 get task") time_now = time.time() self.last_request_time = time_now self.last_active_time = time_now self.request_task(task)