def normal_roundtrip_worker(self, server_address): last_roundtrip_download_size = 0 while self.running: if self.on_road_num > g.config.concurent_thread_num * 0.8: block = True elif last_roundtrip_download_size > g.config.block_max_size: block = False elif len(self.conn_list) > 0 and self.on_road_num < 1: # keep at least one pulling thread block = False elif len(self.conn_list) > 0 and time.time() - self.last_download_data_time < 120 and \ self.on_road_num < g.config.concurent_thread_num * 0.1: # busy, have data download block = False else: block = True if block: get_timeout = 24 * 3600 else: get_timeout = 0 # self.transfer_list[transfer_no]["stat"] = "get local data" upload_data, send_sn = self.upload_task_queue.get(get_timeout) transfer_no = self.get_transfer_no() self.transfer_list[transfer_no] = {} self.transfer_list[transfer_no]["sn"] = send_sn send_data_len = len(upload_data) upload_ack_data = self.ack_pool.get() send_ack_len = len(upload_ack_data) magic = "P" pack_type = 2 if self.on_road_num > g.config.concurent_thread_num * 0.8: server_timeout = 0 else: server_timeout = g.config.roundtrip_timeout / 2 request_session_id = self.session_id upload_data_head = struct.pack("<cBB8sIIBIH", magic, g.protocol_version, pack_type, str(self.session_id), transfer_no, send_sn, server_timeout, send_data_len, send_ack_len) upload_post_buf = base_container.WriteBuffer(upload_data_head) upload_post_buf.append(upload_data) upload_post_buf.append(upload_ack_data) upload_post_data = str(upload_post_buf) upload_post_data = encrypt_data(upload_post_data) try_no = 0 while self.running: try_no += 1 sleep_time = min(try_no, 30) self.last_roundtrip_time = time.time() start_time = time.time() with self.mutex: self.on_road_num += 1 # xlog.debug("start roundtrip transfer_no:%d send_data_len:%d ack_len:%d", transfer_no, send_data_len, send_ack_len) try: self.transfer_list[transfer_no]["try"] = try_no self.transfer_list[transfer_no]["stat"] = "request" self.transfer_list[transfer_no]["start"] = time.time() content, status, response = g.http_client.request(method="POST", host=g.server_host, path="/data", data=upload_post_data, timeout=g.config.roundtrip_timeout) traffic = len(upload_post_data) + len(content) + 645 self.traffic += traffic g.quota -= traffic except Exception as e: xlog.exception("request except:%r retry %d", e, try_no) time.sleep(sleep_time) if transfer_no not in self.transfer_list: break else: continue finally: with self.mutex: self.on_road_num -= 1 if status == 405: # session_id not exist on server if self.running: if self.session_id == request_session_id: xlog.warn("server session_id:%s not exist, reset session.", request_session_id) self.reset() return elif status == 200: recv_len = len(content) if recv_len < 6: xlog.error("roundtrip time:%d transfer_no:%d sn:%d send:%d len:%d status:%r retry:%d", (time.time() - start_time) * 1000, transfer_no, send_sn, send_data_len, len(content), status, try_no) continue content = decrypt_data(content) data = base_container.ReadBuffer(content) magic, version, pack_type = struct.unpack("<cBB", data.get(3)) if magic != "P" or version != g.protocol_version: xlog.error("get data head:%s", utils.str2hex(content[:2])) time.sleep(100) break if pack_type == 3: # error report error_code, message_len = struct.unpack("<BH", data.get(3)) message = data.get(message_len) xlog.warn("error report code:%d, msg:%s", error_code, message) if error_code == 1: # no quota xlog.warn("login x_server error:no quota") self.stop() return else: xlog.error("unknown error code:%d", error_code) return if pack_type != 2: # normal download traffic pack xlog.error("pack type:%d", pack_type) time.sleep(100) break sn, time_cost = struct.unpack("<II", data.get(8)) xlog.debug( "roundtrip time:%d cost:%d transfer_no:%d send_sn:%d send:%d recv_sn:%d rcv:%d status:%r", (time.time() - start_time) * 1000, time_cost, transfer_no, send_sn, send_data_len, sn, len(content), status) data_len = len(data) if (sn > 0 and data_len == 0) or (sn == 0 and data_len > 0): xlog.warn("get sn:%d len:%d %s", sn, data_len, data) if sn: self.last_download_data_time = time.time() last_roundtrip_download_size = data_len # xlog.debug("get sn:%d len:%d", sn, data_len) self.download_order_queue.put(sn, data) ack_pak = struct.pack("<Q", transfer_no) self.ack_pool.put(ack_pak) else: last_roundtrip_download_size = 0 if send_data_len == 0 and data_len > g.config.block_max_size: need_more_thread_num = int(g.config.concurent_thread_num * 0.5 - self.on_road_num) if need_more_thread_num > 0: for j in range(0, need_more_thread_num): if self.on_road_num > g.config.concurent_thread_num * 0.5: break self.touch_roundtrip() break else: xlog.warn("roundtrip time:%d transfer_no:%d send_sn:%d send:%d status:%r retry:%d", (time.time() - start_time) * 1000, transfer_no, send_sn, send_data_len, status, try_no) time.sleep(sleep_time) try: if transfer_no in self.transfer_list: del self.transfer_list[transfer_no] except: pass xlog.info("roundtrip port:%d thread exit", server_address[1])
def normal_roundtrip_worker(self): while self.running: data, ack = self.get_send_data() if not self.running: return send_data_len = len(data) send_ack_len = len(ack) transfer_no = self.get_transfer_no() magic = "P" pack_type = 2 if self.on_road_num > g.config.concurent_thread_num * 0.6: server_timeout = 0 else: server_timeout = g.config.roundtrip_timeout request_session_id = self.session_id upload_data_head = struct.pack("<cBB8sIBIH", magic, g.protocol_version, pack_type, str(self.session_id), transfer_no, server_timeout, send_data_len, send_ack_len) upload_post_buf = base_container.WriteBuffer(upload_data_head) upload_post_buf.append(data) upload_post_buf.append(ack) upload_post_data = str(upload_post_buf) upload_post_data = encrypt_data(upload_post_data) self.last_send_time = time.time() sleep_time = 1 start_time = time.time() with self.lock: self.on_road_num += 1 self.transfer_list[transfer_no] = {} self.transfer_list[transfer_no]["stat"] = "request" self.transfer_list[transfer_no]["start"] = start_time #xlog.debug("start roundtrip transfer_no:%d send_data_len:%d ack_len:%d timeout:%d", # transfer_no, send_data_len, send_ack_len, server_timeout) try: content, status, response = g.http_client.request( method="POST", host=g.server_host, path="/data?tid=%d" % transfer_no, data=upload_post_data, headers={"Content-Length": str(len(upload_post_data))}, timeout=server_timeout + g.config.network_timeout) traffic = len(upload_post_data) + len(content) + 645 self.traffic += traffic g.quota -= traffic except Exception as e: xlog.exception("request except:%r ", e) time.sleep(sleep_time) continue finally: with self.lock: self.on_road_num -= 1 try: if transfer_no in self.transfer_list: del self.transfer_list[transfer_no] except: pass g.stat["roundtrip_num"] += 1 roundtrip_time = (time.time() - start_time) * 1000 if status == 521: xlog.warn("X-tunnel server is down, try get new server.") g.server_host = None self.stop() login_process() return if status != 200: xlog.warn( "roundtrip time:%d transfer_no:%d send:%d status:%r ", roundtrip_time, transfer_no, send_data_len, status) time.sleep(sleep_time) continue recv_len = len(content) if recv_len < 6: xlog.warn( "roundtrip time:%d transfer_no:%d send:%d recv:%d Head", roundtrip_time, transfer_no, send_data_len, recv_len) continue content = decrypt_data(content) payload = base_container.ReadBuffer(content) magic, version, pack_type = struct.unpack("<cBB", payload.get(3)) if magic != "P" or version != g.protocol_version: xlog.warn("get data head:%s", utils.str2hex(content[:2])) time.sleep(sleep_time) continue if pack_type == 3: # error report error_code, message_len = struct.unpack("<BH", payload.get(3)) message = payload.get(message_len) # xlog.warn("report code:%d, msg:%s", error_code, message) if error_code == 1: # no quota xlog.warn("x_server error:no quota") self.stop() return elif error_code == 2: # unpack error xlog.warn( "roundtrip time:%d transfer_no:%d send:%d recv:%d unpack_error:%s", roundtrip_time, transfer_no, send_data_len, len(content), message) continue elif error_code == 3: # session not exist if self.session_id == request_session_id: xlog.warn( "server session_id:%s not exist, reset session.", request_session_id) self.reset() return else: continue else: xlog.error("unknown error code:%d, message:%s", error_code, message) time.sleep(sleep_time) continue if pack_type != 2: # normal download traffic pack xlog.error("pack type:%d", pack_type) time.sleep(100) continue time_cost, server_send_pool_size, data_len, ack_len = struct.unpack( "<IIIH", payload.get(14)) xlog.debug( "roundtrip time:%d cost:%d transfer_no:%d send:%d rcv:%d ", roundtrip_time, time_cost, transfer_no, send_data_len, len(content)) self.trigger_more(server_send_pool_size) rtt = roundtrip_time - time_cost rtt = max(100, rtt) speed = (send_data_len + len(content) + 400) / rtt response.worker.update_debug_data(rtt, send_data_len, len(content), speed) if rtt > 8000: xlog.debug("rtt:%d speed:%d trace:%s", rtt, speed, response.worker.get_trace()) xlog.debug("task trace:%s", response.task.get_trace()) g.stat["slow_roundtrip"] += 1 try: data = payload.get_buf(data_len) ack = payload.get_buf(ack_len) except: xlog.debug("data not enough") continue try: self.roundtrip_process(data, ack) self.last_receive_time = time.time() except Exception as e: xlog.exception("data process:%r", e) continue xlog.info("roundtrip thread exit")