def get_data(self):
time_now = time.time()
buf = base_container.WriteBuffer()
with self.lock:
for sn in self.unacked_send_list:
pk = self.unacked_send_list[sn]
if isinstance(pk, str):
continue
payload, send_time = pk
if time_now - send_time > self.resend_timeout:
g.stat["resend"] += 1
buf.append(self.sn_payload_head(sn, payload))
buf.append(payload)
self.unacked_send_list[sn] = (payload, time_now)
if len(buf) > g.config.max_payload:
return buf
if self.send_buffer.pool_size > g.config.max_payload or \
(self.send_buffer.pool_size > 0 and (
(self.send_buffer.last_put_time - self.last_send_time > self.send_delay) or
(time.time() - self.send_buffer.last_put_time > self.send_delay))):
payload, sn = self.send_buffer.get()
self.unacked_send_list[sn] = (payload, time_now)
buf.append(self.sn_payload_head(sn, payload))
buf.append(payload)
if len(buf) > g.config.max_payload:
return buf
return buf
def send_conn_data(self, conn_id, data, no_delay=False):
if not self.running:
return
# xlog.debug("upload conn_id:%d, len:%d", conn_id, len(data))
buf = base_container.WriteBuffer()
buf.append(struct.pack("<BII", 2, 4 + len(data), conn_id))
buf.append(data)
self.upload_task_queue.put(buf, no_delay)
def get_ack(self, force=False):
time_now = time.time()
if force or \
(self.last_receive_time < self.last_send_time and
time_now - self.last_send_time > self.ack_delay):
buf = base_container.WriteBuffer()
buf.append(struct.pack("<I", self.receive_process.next_sn - 1))
for sn in self.receive_process.block_list:
buf.append(struct.pack("<I", sn))
return buf
return ""
def send_conn_data(self, conn_id, data, no_delay=False):
if not self.running:
return
# xlog.debug("upload conn_id:%d, len:%d", conn_id, len(data))
buf = base_container.WriteBuffer()
buf.append(struct.pack("<II", conn_id, len(data)))
buf.append(data)
self.send_buffer.put(buf)
if self.send_buffer.pool_size > g.config.max_payload or \
time.time() - self.last_send_time > self.send_delay:
# xlog.debug("notify on send conn data")
self.wait_queue.notify()
def normal_roundtrip_worker(self, server_address):
last_roundtrip_download_size = 0
while self.running:
if self.on_road_num > g.config.concurent_thread_num * 0.8:
block = True
elif last_roundtrip_download_size > g.config.block_max_size:
block = False
elif len(self.conn_list) > 0 and self.on_road_num < 1:
# keep at least one pulling thread
block = False
elif len(self.conn_list) > 0 and time.time() - self.last_download_data_time < 120 and \
self.on_road_num < g.config.concurent_thread_num * 0.1:
# busy, have data download
block = False
else:
block = True
if block:
get_timeout = 24 * 3600
else:
get_timeout = 0
# self.transfer_list[transfer_no]["stat"] = "get local data"
upload_data, send_sn = self.upload_task_queue.get(get_timeout)
transfer_no = self.get_transfer_no()
self.transfer_list[transfer_no] = {}
self.transfer_list[transfer_no]["sn"] = send_sn
send_data_len = len(upload_data)
upload_ack_data = self.ack_pool.get()
send_ack_len = len(upload_ack_data)
magic = "P"
pack_type = 2
if self.on_road_num > g.config.concurent_thread_num * 0.8:
server_timeout = 0
else:
server_timeout = g.config.roundtrip_timeout / 2
request_session_id = self.session_id
upload_data_head = struct.pack("<cBB8sIIBIH", magic, g.protocol_version, pack_type, str(self.session_id),
transfer_no,
send_sn, server_timeout, send_data_len, send_ack_len)
upload_post_buf = base_container.WriteBuffer(upload_data_head)
upload_post_buf.append(upload_data)
upload_post_buf.append(upload_ack_data)
upload_post_data = str(upload_post_buf)
upload_post_data = encrypt_data(upload_post_data)
try_no = 0
while self.running:
try_no += 1
sleep_time = min(try_no, 30)
self.last_roundtrip_time = time.time()
start_time = time.time()
with self.mutex:
self.on_road_num += 1
# xlog.debug("start roundtrip transfer_no:%d send_data_len:%d ack_len:%d", transfer_no, send_data_len, send_ack_len)
try:
self.transfer_list[transfer_no]["try"] = try_no
self.transfer_list[transfer_no]["stat"] = "request"
self.transfer_list[transfer_no]["start"] = time.time()
content, status, response = g.http_client.request(method="POST", host=g.server_host,
path="/data", data=upload_post_data,
timeout=g.config.roundtrip_timeout)
traffic = len(upload_post_data) + len(content) + 645
self.traffic += traffic
g.quota -= traffic
except Exception as e:
xlog.exception("request except:%r retry %d", e, try_no)
time.sleep(sleep_time)
if transfer_no not in self.transfer_list:
break
else:
continue
finally:
with self.mutex:
self.on_road_num -= 1
if status == 405: # session_id not exist on server
if self.running:
if self.session_id == request_session_id:
xlog.warn("server session_id:%s not exist, reset session.", request_session_id)
self.reset()
return
elif status == 200:
recv_len = len(content)
if recv_len < 6:
xlog.error("roundtrip time:%d transfer_no:%d sn:%d send:%d len:%d status:%r retry:%d",
(time.time() - start_time) * 1000, transfer_no, send_sn, send_data_len, len(content),
status, try_no)
continue
content = decrypt_data(content)
data = base_container.ReadBuffer(content)
magic, version, pack_type = struct.unpack("<cBB", data.get(3))
if magic != "P" or version != g.protocol_version:
xlog.error("get data head:%s", utils.str2hex(content[:2]))
time.sleep(100)
break
if pack_type == 3: # error report
error_code, message_len = struct.unpack("<BH", data.get(3))
message = data.get(message_len)
xlog.warn("error report code:%d, msg:%s", error_code, message)
if error_code == 1: # no quota
xlog.warn("login x_server error:no quota")
self.stop()
return
else:
xlog.error("unknown error code:%d", error_code)
return
if pack_type != 2: # normal download traffic pack
xlog.error("pack type:%d", pack_type)
time.sleep(100)
break
sn, time_cost = struct.unpack("<II", data.get(8))
xlog.debug(
"roundtrip time:%d cost:%d transfer_no:%d send_sn:%d send:%d recv_sn:%d rcv:%d status:%r",
(time.time() - start_time) * 1000, time_cost, transfer_no, send_sn, send_data_len, sn,
len(content), status)
data_len = len(data)
if (sn > 0 and data_len == 0) or (sn == 0 and data_len > 0):
xlog.warn("get sn:%d len:%d %s", sn, data_len, data)
if sn:
self.last_download_data_time = time.time()
last_roundtrip_download_size = data_len
# xlog.debug("get sn:%d len:%d", sn, data_len)
self.download_order_queue.put(sn, data)
ack_pak = struct.pack("<Q", transfer_no)
self.ack_pool.put(ack_pak)
else:
last_roundtrip_download_size = 0
if send_data_len == 0 and data_len > g.config.block_max_size:
need_more_thread_num = int(g.config.concurent_thread_num * 0.5 - self.on_road_num)
if need_more_thread_num > 0:
for j in range(0, need_more_thread_num):
if self.on_road_num > g.config.concurent_thread_num * 0.5:
break
self.touch_roundtrip()
break
else:
xlog.warn("roundtrip time:%d transfer_no:%d send_sn:%d send:%d status:%r retry:%d",
(time.time() - start_time) * 1000, transfer_no, send_sn, send_data_len, status, try_no)
time.sleep(sleep_time)
try:
if transfer_no in self.transfer_list:
del self.transfer_list[transfer_no]
except:
pass
xlog.info("roundtrip port:%d thread exit", server_address[1])
def normal_roundtrip_worker(self):
while self.running:
data, ack = self.get_send_data()
if not self.running:
return
send_data_len = len(data)
send_ack_len = len(ack)
transfer_no = self.get_transfer_no()
magic = "P"
pack_type = 2
if self.on_road_num > g.config.concurent_thread_num * 0.6:
server_timeout = 0
else:
server_timeout = g.config.roundtrip_timeout
request_session_id = self.session_id
upload_data_head = struct.pack("<cBB8sIBIH", magic,
g.protocol_version, pack_type,
str(self.session_id), transfer_no,
server_timeout, send_data_len,
send_ack_len)
upload_post_buf = base_container.WriteBuffer(upload_data_head)
upload_post_buf.append(data)
upload_post_buf.append(ack)
upload_post_data = str(upload_post_buf)
upload_post_data = encrypt_data(upload_post_data)
self.last_send_time = time.time()
sleep_time = 1
start_time = time.time()
with self.lock:
self.on_road_num += 1
self.transfer_list[transfer_no] = {}
self.transfer_list[transfer_no]["stat"] = "request"
self.transfer_list[transfer_no]["start"] = start_time
#xlog.debug("start roundtrip transfer_no:%d send_data_len:%d ack_len:%d timeout:%d",
# transfer_no, send_data_len, send_ack_len, server_timeout)
try:
content, status, response = g.http_client.request(
method="POST",
host=g.server_host,
path="/data?tid=%d" % transfer_no,
data=upload_post_data,
headers={"Content-Length": str(len(upload_post_data))},
timeout=server_timeout + g.config.network_timeout)
traffic = len(upload_post_data) + len(content) + 645
self.traffic += traffic
g.quota -= traffic
except Exception as e:
xlog.exception("request except:%r ", e)
time.sleep(sleep_time)
continue
finally:
with self.lock:
self.on_road_num -= 1
try:
if transfer_no in self.transfer_list:
del self.transfer_list[transfer_no]
except:
pass
g.stat["roundtrip_num"] += 1
roundtrip_time = (time.time() - start_time) * 1000
if status == 521:
xlog.warn("X-tunnel server is down, try get new server.")
g.server_host = None
self.stop()
login_process()
return
if status != 200:
xlog.warn(
"roundtrip time:%d transfer_no:%d send:%d status:%r ",
roundtrip_time, transfer_no, send_data_len, status)
time.sleep(sleep_time)
continue
recv_len = len(content)
if recv_len < 6:
xlog.warn(
"roundtrip time:%d transfer_no:%d send:%d recv:%d Head",
roundtrip_time, transfer_no, send_data_len, recv_len)
continue
content = decrypt_data(content)
payload = base_container.ReadBuffer(content)
magic, version, pack_type = struct.unpack("<cBB", payload.get(3))
if magic != "P" or version != g.protocol_version:
xlog.warn("get data head:%s", utils.str2hex(content[:2]))
time.sleep(sleep_time)
continue
if pack_type == 3: # error report
error_code, message_len = struct.unpack("<BH", payload.get(3))
message = payload.get(message_len)
# xlog.warn("report code:%d, msg:%s", error_code, message)
if error_code == 1:
# no quota
xlog.warn("x_server error:no quota")
self.stop()
return
elif error_code == 2:
# unpack error
xlog.warn(
"roundtrip time:%d transfer_no:%d send:%d recv:%d unpack_error:%s",
roundtrip_time, transfer_no, send_data_len,
len(content), message)
continue
elif error_code == 3:
# session not exist
if self.session_id == request_session_id:
xlog.warn(
"server session_id:%s not exist, reset session.",
request_session_id)
self.reset()
return
else:
continue
else:
xlog.error("unknown error code:%d, message:%s", error_code,
message)
time.sleep(sleep_time)
continue
if pack_type != 2: # normal download traffic pack
xlog.error("pack type:%d", pack_type)
time.sleep(100)
continue
time_cost, server_send_pool_size, data_len, ack_len = struct.unpack(
"<IIIH", payload.get(14))
xlog.debug(
"roundtrip time:%d cost:%d transfer_no:%d send:%d rcv:%d ",
roundtrip_time, time_cost, transfer_no, send_data_len,
len(content))
self.trigger_more(server_send_pool_size)
rtt = roundtrip_time - time_cost
rtt = max(100, rtt)
speed = (send_data_len + len(content) + 400) / rtt
response.worker.update_debug_data(rtt, send_data_len, len(content),
speed)
if rtt > 8000:
xlog.debug("rtt:%d speed:%d trace:%s", rtt, speed,
response.worker.get_trace())
xlog.debug("task trace:%s", response.task.get_trace())
g.stat["slow_roundtrip"] += 1
try:
data = payload.get_buf(data_len)
ack = payload.get_buf(ack_len)
except:
xlog.debug("data not enough")
continue
try:
self.roundtrip_process(data, ack)
self.last_receive_time = time.time()
except Exception as e:
xlog.exception("data process:%r", e)
continue
xlog.info("roundtrip thread exit")
