def login_session(self): if len(g.server_host) == 0 or g.server_port == 0: return False try: start_time = time.time() magic = "P" pack_type = 1 upload_data_head = struct.pack("<cBB8sIHII", magic, g.protocol_version, pack_type, str(self.session_id), g.config.block_max_size, g.config.send_delay, g.config.windows_size, g.config.windows_ack) upload_data_head += struct.pack("<H", len(g.config.login_account)) + str(g.config.login_account) upload_data_head += struct.pack("<H", len(g.config.login_password)) + str(g.config.login_password) upload_post_data = encrypt_data(upload_data_head) http_client = HTTP_client((g.server_host, g.server_port), g.proxy, g.config.use_https, g.config.conn_life, cert=g.cert) content, status, heads = http_client.request(method="POST", path="data", data=upload_post_data, timeout=g.config.roundtrip_timeout) time_cost = time.time() - start_time if status != 200: g.last_api_error = "session server login fail:%r" % status xlog.warn("login session fail, status:%r", status) return False if len(content) < 6: xlog.error("login data len:%d fail", len(content)) return False info = decrypt_data(content) magic, protocol_version, pack_type, res, message_len = struct.unpack("<cBBBH", info[:6]) message = info[6:] if magic != "P" or protocol_version != 1 or pack_type != 1: xlog.error("login_session time:%d head error:%s", 1000 * time_cost, utils.str2hex(info[:6])) return False if res != 0: g.last_api_error = "session server login fail, code:%d msg:%s" % (res, message) xlog.warn("login_session time:%d fail, res:%d msg:%s", 1000 * time_cost, res, message) return False g.last_api_error = "" xlog.info("login_session time:%d msg:%s", 1000 * time_cost, message) return True except Exception as e: xlog.exception("login_session e:%r", e) return False
def get_api_server_http_client(): api_server = urlparse.urlparse(g.config.api_server) http_client = HTTP_client((api_server.hostname, api_server.port), g.proxy, g.config.use_https, g.config.conn_life, cert=g.cert) return http_client
def normal_roundtrip_worker(self, server_address): last_roundtrip_download_size = 0 http_client = HTTP_client(server_address, g.proxy, g.config.use_https, g.config.conn_life, cert=g.cert) while self.running: if self.on_road_num > g.config.concurent_thread_num * 0.8: block = True elif last_roundtrip_download_size > g.config.block_max_size: block = False elif len(self.conn_list) > 0 and self.on_road_num < 1: # keep at least one pulling thread block = False elif len(self.conn_list) > 0 and time.time() - self.last_download_data_time < 120 and \ self.on_road_num < g.config.concurent_thread_num * 0.1: # busy, have data download block = False else: block = True if block: get_timeout = 24 * 3600 else: get_timeout = 0 # self.transfer_list[transfer_no]["stat"] = "get local data" upload_data, send_sn = self.upload_task_queue.get(get_timeout) transfer_no = self.get_transfer_no() self.transfer_list[transfer_no] = {} self.transfer_list[transfer_no]["sn"] = send_sn send_data_len = len(upload_data) upload_ack_data = self.ack_pool.get() send_ack_len = len(upload_ack_data) magic = "P" pack_type = 2 if self.on_road_num > g.config.concurent_thread_num * 0.8: server_timeout = 0 else: server_timeout = g.config.roundtrip_timeout / 2 upload_data_head = struct.pack("<cBB8sIIBIH", magic, g.protocol_version, pack_type, str(self.session_id), transfer_no, send_sn, server_timeout, send_data_len, send_ack_len) upload_post_buf = base_container.WriteBuffer(upload_data_head) upload_post_buf.append(upload_data) upload_post_buf.append(upload_ack_data) upload_post_data = str(upload_post_buf) upload_post_data = encrypt_data(upload_post_data) try_no = 0 while self.running: try_no += 1 sleep_time = min(try_no, 30) self.last_roundtrip_time = time.time() start_time = time.time() with self.mutex: self.on_road_num += 1 # xlog.debug("start roundtrip transfer_no:%d send_data_len:%d ack_len:%d", transfer_no, send_data_len, send_ack_len) try: self.transfer_list[transfer_no]["try"] = try_no self.transfer_list[transfer_no]["stat"] = "request" self.transfer_list[transfer_no]["start"] = time.time() content, status, response = http_client.request( method="POST", path="data", data=upload_post_data, timeout=g.config.roundtrip_timeout) traffic = len(upload_post_data) + len(content) + 645 self.traffic += traffic g.quota -= traffic except Exception as e: xlog.exception("request except:%r retry %d", e, try_no) time.sleep(sleep_time) continue finally: with self.mutex: self.on_road_num -= 1 if status == 405: # session_id not exist on server if self.running: xlog.warn( "server session_id not exist, start reset session") self.reset() return elif status == 200: recv_len = len(content) if recv_len < 6: xlog.error( "roundtrip time:%d transfer_no:%d sn:%d send:%d status:%r retry:%d", (time.time() - start_time) * 1000, transfer_no, send_sn, send_data_len, len(content), status, try_no) continue content = decrypt_data(content) data = base_container.ReadBuffer(content) magic, version, pack_type = struct.unpack( "<cBB", data.get(3)) if magic != "P" or version != g.protocol_version: xlog.error("get data head:%s", utils.str2hex(content[:2])) time.sleep(100) break if pack_type == 3: # error report error_code, message_len = struct.unpack( "<BH", data.get(3)) message = data.get(message_len) xlog.warn("error report code:%d, msg:%s", error_code, message) if error_code == 1: # no quota xlog.warn("login x_server error:no quota") self.stop() return else: xlog.error("unknown error code:%d", error_code) return if pack_type != 2: # normal download traffic pack xlog.error("pack type:%d", pack_type) time.sleep(100) break sn, time_cost = struct.unpack("<II", data.get(8)) xlog.debug( "roundtrip time:%d cost:%d transfer_no:%d send_sn:%d send:%d recv_sn:%d rcv:%d status:%r", (time.time() - start_time) * 1000, time_cost, transfer_no, send_sn, send_data_len, sn, len(content), status) data_len = len(data) if (sn > 0 and data_len == 0) or (sn == 0 and data_len > 0): xlog.warn("get sn:%d len:%d %s", sn, data_len, data) if sn: self.last_download_data_time = time.time() last_roundtrip_download_size = data_len # xlog.debug("get sn:%d len:%d", sn, data_len) self.download_order_queue.put(sn, data) ack_pak = struct.pack("<Q", transfer_no) self.ack_pool.put(ack_pak) else: last_roundtrip_download_size = 0 if send_data_len == 0 and data_len > g.config.block_max_size: need_more_thread_num = int( g.config.concurent_thread_num * 0.5 - self.on_road_num) if need_more_thread_num > 0: for j in range(0, need_more_thread_num): if self.on_road_num > g.config.concurent_thread_num * 0.5: break self.touch_roundtrip() break else: xlog.warn( "roundtrip time:%d transfer_no:%d send_sn:%d send:%d status:%r retry:%d", (time.time() - start_time) * 1000, transfer_no, send_sn, send_data_len, status, try_no) time.sleep(sleep_time) del self.transfer_list[transfer_no] xlog.info("roundtrip port:%d thread exit", server_address[1])
def normal_roundtrip_worker(self, server_address): last_roundtrip_download_size = 0 http_client = HTTP_client(server_address, g.proxy, g.config.use_https, g.config.conn_life, cert=g.cert) while self.running: if self.on_road_num > g.config.concurent_thread_num * 0.8: block = True elif last_roundtrip_download_size > g.config.block_max_size: block = False elif len(self.conn_list) > 0 and self.on_road_num < 1: # keep at least one pulling thread block = False elif len(self.conn_list) > 0 and time.time() - self.last_download_data_time < 120 and \ self.on_road_num < g.config.concurent_thread_num * 0.1: # busy, have data download block = False else: block = True if block: get_timeout = 24 * 3600 else: get_timeout = 0 # self.transfer_list[transfer_no]["stat"] = "get local data" upload_data, send_sn = self.upload_task_queue.get(get_timeout) transfer_no = self.get_transfer_no() self.transfer_list[transfer_no] = {} self.transfer_list[transfer_no]["sn"] = send_sn send_data_len = len(upload_data) upload_ack_data = self.ack_pool.get() send_ack_len = len(upload_ack_data) magic = "P" pack_type = 2 if self.on_road_num > g.config.concurent_thread_num * 0.8: server_timeout = 0 else: server_timeout = g.config.roundtrip_timeout / 2 upload_data_head = struct.pack("<cBB8sIIBIH", magic, g.protocol_version, pack_type, str(self.session_id), transfer_no, send_sn, server_timeout, send_data_len, send_ack_len) upload_post_buf = base_container.WriteBuffer(upload_data_head) upload_post_buf.append(upload_data) upload_post_buf.append(upload_ack_data) upload_post_data = str(upload_post_buf) upload_post_data = encrypt_data(upload_post_data) try_no = 0 while self.running: try_no += 1 sleep_time = min(try_no, 30) self.last_roundtrip_time = time.time() start_time = time.time() with self.mutex: self.on_road_num += 1 # xlog.debug("start roundtrip transfer_no:%d send_data_len:%d ack_len:%d", transfer_no, send_data_len, send_ack_len) try: self.transfer_list[transfer_no]["try"] = try_no self.transfer_list[transfer_no]["stat"] = "request" self.transfer_list[transfer_no]["start"] = time.time() content, status, response = http_client.request(method="POST", path="data", data=upload_post_data, timeout=g.config.roundtrip_timeout) traffic = len(upload_post_data) + len(content) + 645 self.traffic += traffic g.quota -= traffic except Exception as e: xlog.exception("request except:%r retry %d", e, try_no) time.sleep(sleep_time) continue finally: with self.mutex: self.on_road_num -= 1 if status == 405: # session_id not exist on server if self.running: xlog.warn("server session_id not exist, start reset session") self.reset() return elif status == 200: recv_len = len(content) if recv_len < 6: xlog.error("roundtrip time:%d transfer_no:%d sn:%d send:%d status:%r retry:%d", (time.time() - start_time) * 1000, transfer_no, send_sn, send_data_len, len(content), status, try_no) continue content = decrypt_data(content) data = base_container.ReadBuffer(content) magic, version, pack_type = struct.unpack("<cBB", data.get(3)) if magic != "P" or version != g.protocol_version: xlog.error("get data head:%s", utils.str2hex(content[:2])) time.sleep(100) break if pack_type == 3: # error report error_code, message_len = struct.unpack("<BH", data.get(3)) message = data.get(message_len) xlog.warn("error report code:%d, msg:%s", error_code, message) if error_code == 1: # no quota xlog.warn("login x_server error:no quota") self.stop() return else: xlog.error("unknown error code:%d", error_code) return if pack_type != 2: # normal download traffic pack xlog.error("pack type:%d", pack_type) time.sleep(100) break sn, time_cost = struct.unpack("<II", data.get(8)) xlog.debug( "roundtrip time:%d cost:%d transfer_no:%d send_sn:%d send:%d recv_sn:%d rcv:%d status:%r", (time.time() - start_time) * 1000, time_cost, transfer_no, send_sn, send_data_len, sn, len(content), status) data_len = len(data) if (sn > 0 and data_len == 0) or (sn == 0 and data_len > 0): xlog.warn("get sn:%d len:%d %s", sn, data_len, data) if sn: self.last_download_data_time = time.time() last_roundtrip_download_size = data_len # xlog.debug("get sn:%d len:%d", sn, data_len) self.download_order_queue.put(sn, data) ack_pak = struct.pack("<Q", transfer_no) self.ack_pool.put(ack_pak) else: last_roundtrip_download_size = 0 if send_data_len == 0 and data_len > g.config.block_max_size: need_more_thread_num = int(g.config.concurent_thread_num * 0.5 - self.on_road_num) if need_more_thread_num > 0: for j in range(0, need_more_thread_num): if self.on_road_num > g.config.concurent_thread_num * 0.5: break self.touch_roundtrip() break else: xlog.warn("roundtrip time:%d transfer_no:%d send_sn:%d send:%d status:%r retry:%d", (time.time() - start_time) * 1000, transfer_no, send_sn, send_data_len, status, try_no) time.sleep(sleep_time) del self.transfer_list[transfer_no] xlog.info("roundtrip port:%d thread exit", server_address[1])