def ack_process(self, ack):
self.lock.acquire()
try:
last_ack = struct.unpack("<I", ack.get(4))[0]
while len(ack):
sn = struct.unpack("<I", ack.get(4))[0]
# xlog.debug("ack: %d", sn)
if sn in self.unacked_send_list:
self.unacked_send_list[sn] = "acked"
for sn in self.unacked_send_list:
if sn > last_ack:
continue
if self.unacked_send_list[sn] == "acked":
continue
# xlog.debug("last_ack:%d sn:%d", last_ack, sn)
self.unacked_send_list[sn] = "acked"
while (self.acked_send_continue_sn + 1) in self.unacked_send_list and \
self.unacked_send_list[self.acked_send_continue_sn + 1] == "acked":
self.acked_send_continue_sn += 1
del self.unacked_send_list[self.acked_send_continue_sn]
except Exception as e:
xlog.exception("ack_process:%r", e)
finally:
self.lock.release()
def ack_process(self, ack):
self.lock.acquire()
try:
last_ack = struct.unpack("<I", ack.get(4))[0]
while len(ack):
sn = struct.unpack("<I", ack.get(4))[0]
# xlog.debug("ack: %d", sn)
if sn in self.wait_ack_send_list:
self.wait_ack_send_list[sn] = "acked"
for sn in self.wait_ack_send_list:
if sn > last_ack:
continue
if self.wait_ack_send_list[sn] == "acked":
continue
# xlog.debug("last_ack:%d sn:%d", last_ack, sn)
self.wait_ack_send_list[sn] = "acked"
while (self.ack_send_continue_sn + 1) in self.wait_ack_send_list and \
self.wait_ack_send_list[self.ack_send_continue_sn + 1] == "acked":
self.ack_send_continue_sn += 1
del self.wait_ack_send_list[self.ack_send_continue_sn]
except Exception as e:
xlog.exception("ack_process:%r", e)
finally:
self.lock.release()
def parse_body(self, data):
self.body_len = len(data)
self.max_age, self.port, protocol_id_length = struct.unpack("!LHxB", data[:8])
pos = 8
self.protocol_id = data[pos:pos+protocol_id_length].tobytes()
pos += protocol_id_length
host_length = struct.unpack("!B", data[pos:pos+1])[0]
pos += 1
self.host = data[pos:pos+host_length].tobytes()
pos += host_length
self.parse_origin(data[pos:])
def cmd_processor(self):
while self.running:
data = self.get_cmd_data()
if not data:
break
cmd_id = struct.unpack("<B", data.get(1))[0]
if cmd_id == 1: # data
self.send_to_sock(data)
elif cmd_id == 3: # ack:
position = struct.unpack("<Q", data.get(8))[0]
self.xlog.debug("Conn session:%s conn:%d ACK:%d", self.session.session_id, self.conn_id, position)
if position > self.remote_acked_position:
self.remote_acked_position = position
self.recv_notice.acquire()
self.recv_notice.notify()
self.recv_notice.release()
elif cmd_id == 2: # Closed
dat = data.get()
if isinstance(dat, memoryview):
dat = dat.tobytes()
self.xlog.debug("Conn session:%s conn:%d Peer Close:%s", self.session.session_id, self.conn_id, dat)
if self.is_client:
self.transfer_peer_close("finish")
self.stop("peer close")
elif cmd_id == 0: # Create connect
if self.port or len(self.host) or self.next_cmd_seq != 1 or self.sock:
raise Exception("put_send_data %s conn:%d Create but host:%s port:%d next seq:%d" % (
self.session.session_id, self.conn_id,
self.host, self.port, self.next_cmd_seq))
self.sock_type = struct.unpack("<B", data.get(1))[0]
host_len = struct.unpack("<H", data.get(2))[0]
self.host = data.get(host_len)
self.port = struct.unpack("<H", data.get(2))[0]
sock, res = self.do_connect(self.host, self.port)
if res is False:
self.xlog.debug("Conn session:%s conn:%d %s:%d Create fail", self.session.session_id, self.conn_id,
self.host, self.port)
self.transfer_peer_close("connect fail")
else:
self.xlog.info("Conn session:%s conn:%d %s:%d", self.session.session_id, self.conn_id, self.host,
self.port)
self.sock = sock
self.recv_thread = threading.Thread(target=self.recv_worker)
self.recv_thread.start()
else:
self.xlog.error("Conn session:%s conn:%d unknown cmd_id:%d",
self.session.session_id, self.conn_id, cmd_id)
raise Exception("put_send_data unknown cmd_id:%d" % cmd_id)
def cmd_processor(self):
while self.running:
data = self.get_cmd_data()
if not data:
break
cmd_id = struct.unpack("<B", data.get(1))[0]
if cmd_id == 1: # data
self.send_to_sock(data)
elif cmd_id == 3: # ack:
position = struct.unpack("<Q", data.get(8))[0]
self.xlog.debug("Conn session:%s conn:%d ACK:%d", self.session.session_id, self.conn_id, position)
if position > self.remote_acked_position:
self.remote_acked_position = position
self.recv_notice.acquire()
self.recv_notice.notify()
self.recv_notice.release()
elif cmd_id == 2: # Closed
dat = data.get()
if isinstance(dat, memoryview):
dat = dat.tobytes()
self.xlog.debug("Conn session:%s conn:%d Peer Close:%s", self.session.session_id, self.conn_id, dat)
if self.is_client:
self.transfer_peer_close("finish")
self.stop("peer close")
elif cmd_id == 0: # Create connect
if self.port or len(self.host) or self.next_cmd_seq != 1 or self.sock:
raise Exception("put_send_data %s conn:%d Create but host:%s port:%d next seq:%d" % (
self.session.session_id, self.conn_id,
self.host, self.port, self.next_cmd_seq))
self.sock_type = struct.unpack("<B", data.get(1))[0]
host_len = struct.unpack("<H", data.get(2))[0]
self.host = data.get(host_len)
self.port = struct.unpack("<H", data.get(2))[0]
sock, res = self.do_connect(self.host, self.port)
if res is False:
self.xlog.debug("Conn session:%s conn:%d %s:%d Create fail", self.session.session_id, self.conn_id,
self.host, self.port)
self.transfer_peer_close("connect fail")
else:
self.xlog.info("Conn session:%s conn:%d %s:%d", self.session.session_id, self.conn_id, self.host,
self.port)
self.sock = sock
self.recv_thread = threading.Thread(target=self.recv_worker)
self.recv_thread.start()
else:
self.xlog.error("Conn session:%s conn:%d unknown cmd_id:%d",
self.session.session_id, self.conn_id, cmd_id)
raise Exception("put_send_data unknown cmd_id:%d" % cmd_id)
def parse_body(self, data):
self.body_len = len(data)
self.max_age, self.port, protocol_id_length = struct.unpack(
"!LHxB", data[:8])
pos = 8
self.protocol_id = data[pos:pos + protocol_id_length].tobytes()
pos += protocol_id_length
host_length = struct.unpack("!B", data[pos:pos + 1])[0]
pos += 1
self.host = data[pos:pos + host_length].tobytes()
pos += host_length
self.parse_origin(data[pos:])
def struct_callback(s):
(name, id, description), s = unpack('SIS', s)
if ObjectParamsStructDesc.has_key(id):
output_extra = {}
for substruct, ename, edescription in ObjectParamsStructDesc[id]:
o, s = unpack(substruct, s)
output_extra[ename] = o
output = [name, id, description, output_extra]
else:
output = [name, id, description, {}]
return output, s
def unpack_response(response):
try:
data = response.task.read(size=2)
if not data:
raise GAE_Exception(600, "get protocol head fail")
if len(data) !=2:
raise GAE_Exception(600, "get protocol head fail, data:%s, len:%d" % (data, len(data)))
headers_length, = struct.unpack('!h', data)
data = response.task.read(size=headers_length)
if not data:
raise GAE_Exception(600,
"get protocol head fail, len:%d" % headers_length)
raw_response_line, headers_data = inflate(data).split('\r\n', 1)
_, status, reason = raw_response_line.split(None, 2)
response.app_status = int(status)
response.app_reason = reason.strip()
headers_block, app_msg = headers_data.split('\r\n\r\n')
headers_pairs = headers_block.split('\r\n')
response.headers = {}
for pair in headers_pairs:
if not pair:
break
k, v = pair.split(': ', 1)
response.headers[k] = v
response.app_msg = app_msg
return response
except Exception as e:
response.worker.close("unpack protocol error")
raise GAE_Exception(600, "unpack protocol:%r at:%s" % (e, traceback.format_exc()))
def __process__(self, data, **kw):
# Unpack the first lot of data
args, leftover = unpack(self.struct, data)
self.__init__(self.sequence, *args, **kw)
if hasattr(self.__class__, "__process__"):
self.__class__.__process__(self, leftover, **kw)
return
# Unpack the second lot of data
try:
moreargs, leftover2 = unpack(self.substruct, leftover)
if len(leftover2) > 0:
raise ValueError("\nError when processing %s.\nClass Structure: %s, Given Data: %r\nExtra data found: %r " % (self.__class__, self.substruct, leftover, leftover2))
except TypeError, e:
raise ValueError("\nError when processing %s.\nClass Structure: %s, Given Data: %r\nNot enough data found: %s" % (self.__class__, self.substruct, leftover, e))
def parse_priority_data(self, data):
MASK = 0x80000000
self.depends_on, self.stream_weight = struct.unpack(
"!LB", data[:5]
)
self.exclusive = bool(self.depends_on & MASK)
self.depends_on &= ~MASK
return 5
def round_trip_process(self, data, ack):
while len(data):
sn, plen = struct.unpack("<II", data.get(8))
pdata = data.get_buf(plen)
# xlog.debug("download sn:%d len:%d", sn, plen)
self.receive_process.put(sn, pdata)
self.ack_process(ack)
def fromstr(cls, data):
"""\
Look at the packet type and morph this object into the
correct type.
"""
args, extra = unpack(Header.struct, data)
if len(extra) > 0:
raise ValueError("Got too much data! %s bytes remaining" % len(extra))
return cls(*args)
def roundtrip_process(self, data, ack):
while len(data):
sn, plen = struct.unpack("<II", data.get(8))
pdata = data.get_buf(plen)
# xlog.debug("upload sn:%d len:%d", sn, plen)
self.receive_process.put(sn, pdata)
self.last_upload_time = time.time()
self.ack_process(ack)
def unpack(self, s):
"""
unpack() -> values, leftover
Returns the unpacked values and any data that is left over.
"""
a, s = xstruct.unpack(self.xstruct, s)
if len(a) != 1:
raise TypeError("WTF?")
return a[0], s
def download_data_processor(self, data):
try:
while len(data):
conn_id, payload_len = struct.unpack("<II", data.get(8))
payload = data.get_buf(payload_len)
# xlog.debug("conn:%d upload data len:%d", conn_id, len(payload))
if conn_id not in self.conn_list:
xlog.debug("conn:%d not exist", conn_id)
continue
self.conn_list[conn_id].put_cmd_data(payload)
except Exception as e:
xlog.exception("download_data_processor:%r", e)
def download_data_processor(self, data):
try:
while len(data):
conn_id, payload_len = struct.unpack("<II", data.get(8))
payload = data.get_buf(payload_len)
if conn_id not in self.conn_list:
#xlog.debug("DATA conn_id %d not in list", conn_id)
continue
# xlog.debug("conn:%d upload data len:%d", conn_id, len(payload))
self.conn_list[conn_id].put_cmd_data(payload)
except Exception as e:
xlog.exception("download_data_processor:%r", e)
def put_cmd_data(self, data):
with self.cmd_notice:
seq = struct.unpack("<I", data.get(4))[0]
if seq < self.next_cmd_seq:
# xlog.warn("put_send_data %s conn:%d seq:%d next:%d",
# self.session.session_id, self.conn_id,
# seq, self.next_cmd_seq)
return
self.cmd_queue[seq] = data.get_buf()
if seq == self.next_cmd_seq:
self.cmd_notice.notify()
def put_cmd_data(self, data):
with self.cmd_notice:
seq = struct.unpack("<I", data.get(4))[0]
if seq < self.next_cmd_seq:
# xlog.warn("put_send_data %s conn:%d seq:%d next:%d",
# self.session.session_id, self.conn_id,
# seq, self.next_cmd_seq)
return
self.cmd_queue[seq] = data.get_buf()
if seq == self.next_cmd_seq:
self.cmd_notice.notify()
def parse_frame_header(header):
"""
Takes a 9-byte frame header and returns a tuple of the appropriate
Frame object and the length that needs to be read from the socket.
"""
fields = struct.unpack("!HBBBL", header)
# First 24 bits are frame length.
length = (fields[0] << 8) + fields[1]
type = fields[2]
flags = fields[3]
stream_id = fields[4]
if type not in FRAMES:
raise ValueError("Unknown frame type %d" % type)
frame = FRAMES[type](stream_id)
frame.parse_flags(flags)
return frame, length
def parse_data(data):
if len(data) == 0:
return ""
o = ""
data = bytes(data)
data = base_container.ReadBuffer(data)
while len(data):
sn, block_len = struct.unpack("<II", data.get(8))
block = data.get_buf(block_len)
o += "sn:%d {" % sn
while len(block):
conn_id, payload_len = struct.unpack("<II", block.get(8))
o += "conn:%d [" % conn_id
conn_data = block.get_buf(payload_len)
seq = struct.unpack("<I", conn_data.get(4))[0]
cmd_id = struct.unpack("<B", conn_data.get(1))[0]
conn_payload = conn_data.get_buf()
if cmd_id == 0: # create connection
sock_type = struct.unpack("<B", conn_payload.get(1))[0]
host_len = struct.unpack("<H", conn_payload.get(2))[0]
host = str(bytes(conn_payload.get(host_len)))
port = struct.unpack("<H", conn_payload.get(2))[0]
o += "%d|Connect:%s:%d" % (seq, host, port)
elif cmd_id == 1: # data
o += "%d|D:%d" % (seq, len(conn_payload))
elif cmd_id == 2: # closed
o += "%d|Closed:%s" % (seq, conn_payload)
elif cmd_id == 3: # ack
position = struct.unpack("<Q", conn_payload.get())[0]
o += "%d|Ack:%d" % (seq, position)
o += "],"
o += "},"
return o
def parse_frame_header(header):
"""
Takes a 9-byte frame header and returns a tuple of the appropriate
Frame object and the length that needs to be read from the socket.
"""
fields = struct.unpack("!HBBBL", header)
# First 24 bits are frame length.
length = (fields[0] << 8) + fields[1]
type = fields[2]
flags = fields[3]
stream_id = fields[4]
if type not in FRAMES:
raise ValueError("Unknown frame type %d" % type)
frame = FRAMES[type](stream_id)
frame.parse_flags(flags)
return frame, length
def parse_body(self, data):
padding_data_length = self.parse_padding_data(data)
self.promised_stream_id = struct.unpack("!L", data[padding_data_length:padding_data_length + 4])[0]
self.data = data[padding_data_length + 4:].tobytes()
self.body_len = len(data)
def login_session(self):
if len(g.server_host) == 0:
return False
start_time = time.time()
while time.time() - start_time < 30:
try:
magic = b"P"
pack_type = 1
upload_data_head = struct.pack("<cBB8sIHIIHH", magic, g.protocol_version, pack_type,
bytes(self.session_id),
g.config.max_payload, g.config.send_delay, g.config.windows_size,
int(g.config.windows_ack), g.config.resend_timeout, g.config.ack_delay)
upload_data_head += struct.pack("<H", len(g.config.login_account)) + bytes(g.config.login_account,
encoding='iso-8859-1')
upload_data_head += struct.pack("<H", len(g.config.login_password)) + bytes(g.config.login_password,
encoding='iso-8859-1')
upload_post_data = encrypt_data(upload_data_head)
content, status, response = g.http_client.request(method="POST", host=g.server_host, path="/data",
data=upload_post_data,
timeout=g.config.network_timeout)
time_cost = time.time() - start_time
if status == 521:
g.last_api_error = "session server is down."
xlog.warn("login session server is down, try get new server.")
g.server_host = None
return False
if status != 200:
g.last_api_error = "session server login fail:%r" % status
xlog.warn("login session fail, status:%r", status)
continue
if len(content) < 6:
g.last_api_error = "session server protocol fail, login res len:%d" % len(content)
xlog.error("login data len:%d fail", len(content))
continue
info = decrypt_data(content)
magic, protocol_version, pack_type, res, message_len = struct.unpack("<cBBBH", info[:6])
message = info[6:]
if isinstance(message, memoryview):
message = message.tobytes()
if magic != b"P" or protocol_version != g.protocol_version or pack_type != 1:
xlog.error("login_session time:%d head error:%s", 1000 * time_cost, utils.str2hex(info[:6]))
return False
if res != 0:
g.last_api_error = "session server login fail, code:%d msg:%s" % (res, message)
xlog.warn("login_session time:%d fail, res:%d msg:%s", 1000 * time_cost, res, message)
return False
g.last_api_error = ""
xlog.info("login_session %s time:%d msg:%s", self.session_id, 1000 * time_cost, message)
return True
except Exception as e:
xlog.exception("login_session e:%r", e)
time.sleep(1)
return False
def parse_body(self, data):
self.last_stream_id, self.error_code = struct.unpack("!LL", data[:8])
self.body_len = len(data)
if len(data) > 8:
self.additional_data = data[8:].tobytes()
def parse_body(self, data):
self.window_increment = struct.unpack("!L", data)[0]
self.body_len = len(data)
def __process__(self, data):
args, leftover = unpack(self.struct, data)
if len(leftover) > 0:
raise ValueError("Left over data found for %r: '%r'" % (self.__class__.__name__, leftover))
self.__init__(self.sequence, *args)
def parse_body(self, data):
padding_data_length = self.parse_padding_data(data)
self.promised_stream_id = struct.unpack(
"!L", data[padding_data_length:padding_data_length + 4])[0]
self.data = data[padding_data_length + 4:].tobytes()
self.body_len = len(data)
def normal_round_trip_worker(self, work_id):
while self.running:
data, ack = self.get_send_data(work_id)
if not self.running:
return
send_data_len = len(data)
send_ack_len = len(ack)
transfer_no = self.get_transfer_no()
# xlog.debug("trip:%d no:%d send data:%s", work_id, transfer_no, parse_data(data))
magic = b"P"
pack_type = 2
if self.send_buffer.pool_size > g.config.max_payload or \
(self.send_buffer.pool_size and len(self.wait_queue.waiters) < g.config.min_on_road):
server_timeout = 0
elif work_id > g.config.concurent_thread_num * 0.9:
server_timeout = 1
elif work_id > g.config.concurent_thread_num * 0.7:
server_timeout = 3
else:
server_timeout = g.config.roundtrip_timeout
request_session_id = self.session_id
upload_data_head = struct.pack("<cBB8sIBIH", magic, g.protocol_version, pack_type,
bytes(self.session_id), transfer_no,
server_timeout, send_data_len, send_ack_len)
upload_post_buf = base_container.WriteBuffer(upload_data_head)
upload_post_buf.append(data)
upload_post_buf.append(ack)
upload_post_data = upload_post_buf.to_bytes()
upload_post_data = encrypt_data(upload_post_data)
self.last_send_time = time.time()
sleep_time = 1
start_time = time.time()
with self.lock:
self.on_road_num += 1
self.transfer_list[transfer_no] = {}
self.transfer_list[transfer_no]["stat"] = "request"
self.transfer_list[transfer_no]["start"] = start_time
# xlog.debug("start trip transfer_no:%d send_data_len:%d ack_len:%d timeout:%d",
# transfer_no, send_data_len, send_ack_len, server_timeout)
try:
content, status, response = g.http_client.request(method="POST", host=g.server_host,
path="/data?tid=%d" % transfer_no,
data=upload_post_data,
headers={
"Content-Length": str(len(upload_post_data))},
timeout=server_timeout + g.config.network_timeout)
traffic = len(upload_post_data) + len(content) + 645
self.traffic += traffic
g.quota -= traffic
if g.quota < 0:
g.quota = 0
except Exception as e:
xlog.exception("request except:%r ", e)
time.sleep(sleep_time)
continue
finally:
with self.lock:
self.on_road_num -= 1
try:
if transfer_no in self.transfer_list:
del self.transfer_list[transfer_no]
except:
pass
g.stat["roundtrip_num"] += 1
roundtrip_time = (time.time() - start_time)
if status == 521:
xlog.warn("X-tunnel server is down, try get new server.")
g.server_host = None
self.stop()
login_process()
return
if status != 200:
xlog.warn("roundtrip time:%f transfer_no:%d send:%d status:%r ",
roundtrip_time, transfer_no, send_data_len, status)
time.sleep(sleep_time)
continue
recv_len = len(content)
if recv_len < 6:
xlog.warn("roundtrip time:%f transfer_no:%d send:%d recv:%d Head",
roundtrip_time, transfer_no, send_data_len, recv_len)
continue
content = decrypt_data(content)
payload = base_container.ReadBuffer(content)
magic, version, pack_type = struct.unpack("<cBB", payload.get(3))
if magic != b"P" or version != g.protocol_version:
xlog.warn("get data head:%s", utils.str2hex(content[:2]))
time.sleep(sleep_time)
continue
if pack_type == 3: # error report
error_code, message_len = struct.unpack("<BH", payload.get(3))
message = payload.get(message_len)
# xlog.warn("report code:%d, msg:%s", error_code, message)
if error_code == 1:
# no quota
xlog.warn("x_server error:no quota")
self.stop()
return
elif error_code == 2:
# unpack error
xlog.warn("roundtrip time:%f transfer_no:%d send:%d recv:%d unpack_error:%s",
roundtrip_time, transfer_no, send_data_len, len(content), message)
continue
elif error_code == 3:
# session not exist
if self.session_id == request_session_id:
xlog.warn("server session_id:%s not exist, reset session.", request_session_id)
self.reset()
return
else:
continue
else:
xlog.error("unknown error code:%d, message:%s", error_code, message)
time.sleep(sleep_time)
continue
if pack_type != 2: # normal download traffic pack
xlog.error("pack type:%d", pack_type)
time.sleep(100)
continue
time_cost, server_send_pool_size, data_len, ack_len = struct.unpack("<IIIH", payload.get(14))
xlog.debug(
"trip:%d no:%d tc:%f cost:%f to:%d snd:%d rcv:%d s_pool:%d on_road:%d target:%d",
work_id, transfer_no,
roundtrip_time, time_cost / 1000.0, server_timeout,
send_data_len, len(content), server_send_pool_size,
self.on_road_num,
self.target_on_roads)
if len(self.conn_list) == 0:
self.target_on_roads = 0
elif len(content) >= g.config.max_payload:
self.target_on_roads = \
min(g.config.concurent_thread_num - g.config.min_on_road, self.target_on_roads + 10)
elif len(content) <= 21:
self.target_on_roads = max(g.config.min_on_road, self.target_on_roads - 5)
self.trigger_more()
rtt = roundtrip_time * 1000 - time_cost
rtt = max(100, rtt)
speed = (send_data_len + len(content) + 400) / rtt
response.worker.update_debug_data(rtt, send_data_len, len(content), speed)
if rtt > 8000:
xlog.debug("rtt:%d speed:%d trace:%s", rtt, speed, response.worker.get_trace())
xlog.debug("task trace:%s", response.task.get_trace())
g.stat["slow_roundtrip"] += 1
try:
data = payload.get_buf(data_len)
ack = payload.get_buf(ack_len)
except Exception as e:
xlog.warn("trip:%d no:%d data not enough %r", work_id, transfer_no, e)
continue
# xlog.debug("trip:%d no:%d recv data:%s", work_id, transfer_no, parse_data(data))
try:
self.round_trip_process(data, ack)
self.last_receive_time = time.time()
except Exception as e:
xlog.exception("data process:%r", e)
xlog.info("roundtrip thread exit")
def login_session(self):
if len(g.server_host) == 0:
return False
start_time = time.time()
while time.time() - start_time < 30:
try:
start_time = time.time()
magic = "P"
pack_type = 1
upload_data_head = struct.pack("<cBB8sIHIIHH", magic, g.protocol_version, pack_type, str(self.session_id),
g.config.max_payload, g.config.send_delay, g.config.windows_size,
g.config.windows_ack, g.config.resend_timeout, g.config.ack_delay)
upload_data_head += struct.pack("<H", len(g.config.login_account)) + str(g.config.login_account)
upload_data_head += struct.pack("<H", len(g.config.login_password)) + str(g.config.login_password)
upload_post_data = encrypt_data(upload_data_head)
content, status, response = g.http_client.request(method="POST", host=g.server_host, path="/data",
data=upload_post_data,
timeout=g.config.network_timeout)
time_cost = time.time() - start_time
if status == 521:
g.last_api_error = "session server is down."
xlog.warn("login session server is down, try get new server.")
g.server_host = None
return False
if status != 200:
g.last_api_error = "session server login fail:%r" % status
xlog.warn("login session fail, status:%r", status)
continue
if len(content) < 6:
g.last_api_error = "session server protocol fail, login res len:%d" % len(content)
xlog.error("login data len:%d fail", len(content))
continue
info = decrypt_data(content)
magic, protocol_version, pack_type, res, message_len = struct.unpack("<cBBBH", info[:6])
message = info[6:]
if isinstance(message, memoryview):
message = message.tobytes()
if magic != "P" or protocol_version != g.protocol_version or pack_type != 1:
xlog.error("login_session time:%d head error:%s", 1000 * time_cost, utils.str2hex(info[:6]))
return False
if res != 0:
g.last_api_error = "session server login fail, code:%d msg:%s" % (res, message)
xlog.warn("login_session time:%d fail, res:%d msg:%s", 1000 * time_cost, res, message)
return False
g.last_api_error = ""
xlog.info("login_session %s time:%d msg:%s", self.session_id, 1000 * time_cost, message)
return True
except Exception as e:
xlog.exception("login_session e:%r", e)
time.sleep(1)
return False
def normal_roundtrip_worker(self):
while self.running:
data, ack = self.get_send_data()
if not self.running:
return
send_data_len = len(data)
send_ack_len = len(ack)
transfer_no = self.get_transfer_no()
magic = "P"
pack_type = 2
if self.on_road_num > g.config.concurent_thread_num * 0.6:
server_timeout = 0
else:
server_timeout = g.config.roundtrip_timeout
request_session_id = self.session_id
upload_data_head = struct.pack("<cBB8sIBIH", magic, g.protocol_version, pack_type,
str(self.session_id), transfer_no,
server_timeout, send_data_len, send_ack_len)
upload_post_buf = base_container.WriteBuffer(upload_data_head)
upload_post_buf.append(data)
upload_post_buf.append(ack)
upload_post_data = str(upload_post_buf)
upload_post_data = encrypt_data(upload_post_data)
self.last_send_time = time.time()
sleep_time = 1
start_time = time.time()
with self.lock:
self.on_road_num += 1
self.transfer_list[transfer_no] = {}
self.transfer_list[transfer_no]["stat"] = "request"
self.transfer_list[transfer_no]["start"] = start_time
#xlog.debug("start roundtrip transfer_no:%d send_data_len:%d ack_len:%d timeout:%d",
# transfer_no, send_data_len, send_ack_len, server_timeout)
try:
content, status, response = g.http_client.request(method="POST", host=g.server_host,
path="/data?tid=%d" % transfer_no,
data=upload_post_data,
headers={"Content-Length": str(len(upload_post_data))},
timeout=server_timeout + g.config.network_timeout)
traffic = len(upload_post_data) + len(content) + 645
self.traffic += traffic
g.quota -= traffic
except Exception as e:
xlog.exception("request except:%r ", e)
time.sleep(sleep_time)
continue
finally:
with self.lock:
self.on_road_num -= 1
try:
if transfer_no in self.transfer_list:
del self.transfer_list[transfer_no]
except:
pass
g.stat["roundtrip_num"] += 1
roundtrip_time = (time.time() - start_time) * 1000
if status == 521:
xlog.warn("X-tunnel server is down, try get new server.")
g.server_host = None
self.stop()
login_process()
return
if status != 200:
xlog.warn("roundtrip time:%d transfer_no:%d send:%d status:%r ",
roundtrip_time, transfer_no, send_data_len, status)
time.sleep(sleep_time)
continue
recv_len = len(content)
if recv_len < 6:
xlog.warn("roundtrip time:%d transfer_no:%d send:%d recv:%d Head",
roundtrip_time, transfer_no, send_data_len, recv_len)
continue
content = decrypt_data(content)
payload = base_container.ReadBuffer(content)
magic, version, pack_type = struct.unpack("<cBB", payload.get(3))
if magic != "P" or version != g.protocol_version:
xlog.warn("get data head:%s", utils.str2hex(content[:2]))
time.sleep(sleep_time)
continue
if pack_type == 3: # error report
error_code, message_len = struct.unpack("<BH", payload.get(3))
message = payload.get(message_len)
# xlog.warn("report code:%d, msg:%s", error_code, message)
if error_code == 1:
# no quota
xlog.warn("x_server error:no quota")
self.stop()
return
elif error_code == 2:
# unpack error
xlog.warn("roundtrip time:%d transfer_no:%d send:%d recv:%d unpack_error:%s",
roundtrip_time, transfer_no, send_data_len, len(content), message)
continue
elif error_code == 3:
# session not exist
if self.session_id == request_session_id:
xlog.warn("server session_id:%s not exist, reset session.", request_session_id)
self.reset()
return
else:
continue
else:
xlog.error("unknown error code:%d, message:%s", error_code, message)
time.sleep(sleep_time)
continue
if pack_type != 2: # normal download traffic pack
xlog.error("pack type:%d", pack_type)
time.sleep(100)
continue
time_cost, server_send_pool_size, data_len, ack_len = struct.unpack("<IIIH", payload.get(14))
xlog.debug("roundtrip time:%d cost:%d transfer_no:%d send:%d rcv:%d ",
roundtrip_time, time_cost, transfer_no, send_data_len, len(content))
self.trigger_more(server_send_pool_size)
rtt = roundtrip_time - time_cost
rtt = max(100, rtt)
speed = (send_data_len + len(content) + 400) / rtt
response.worker.update_debug_data(rtt, send_data_len, len(content), speed)
if rtt > 8000:
xlog.debug("rtt:%d speed:%d trace:%s", rtt, speed, response.worker.get_trace())
xlog.debug("task trace:%s", response.task.get_trace())
g.stat["slow_roundtrip"] += 1
try:
data = payload.get_buf(data_len)
ack = payload.get_buf(ack_len)
except:
xlog.debug("data not enough")
continue
try:
self.roundtrip_process(data, ack)
self.last_receive_time = time.time()
except Exception as e:
xlog.exception("data process:%r", e)
continue
xlog.info("roundtrip thread exit")
def parse_padding_data(self, data):
if b'PADDED' in self.flags:
self.pad_length = struct.unpack('!B', data[:1])[0]
return 1
return 0
def parse_body(self, data):
self.window_increment = struct.unpack("!L", data)[0]
self.body_len = len(data)
def parse_body(self, data):
self.last_stream_id, self.error_code = struct.unpack("!LL", data[:8])
self.body_len = len(data)
if len(data) > 8:
self.additional_data = data[8:].tobytes()
def parse_body(self, data):
for i in range(0, len(data), 6):
name, value = struct.unpack("!HL", data[i:i+6])
self.settings[name] = value
self.body_len = len(data)
def unpack(self, s):
a, s = xstruct.unpack(self.xstruct, s)
if len(a) != 1:
raise TypeError("WTF?")
return a[0], s
def parse_body(self, data):
if len(data) != 4:
raise ValueError()
self.error_code = struct.unpack("!L", data)[0]
self.body_len = len(data)
def __process__(self, data):
try:
args, leftover = unpack(self.struct, data, callback=self.struct_callback)
except TypeError, e:
raise TypeError("Problem when trying to unpack %s (%s) %s" % (self.__class__, self.struct, e))
def parse_body(self, data):
if len(data) != 4:
raise ValueError()
self.error_code = struct.unpack("!L", data)[0]
self.body_len = len(data)
def parse_priority_data(self, data):
MASK = 0x80000000
self.depends_on, self.stream_weight = struct.unpack("!LB", data[:5])
self.exclusive = bool(self.depends_on & MASK)
self.depends_on &= ~MASK
return 5
def parse_body(self, data):
for i in range(0, len(data), 6):
name, value = struct.unpack("!HL", data[i:i + 6])
self.settings[name] = value
self.body_len = len(data)
def parse_padding_data(self, data):
if 'PADDED' in self.flags:
self.pad_length = struct.unpack('!B', data[:1])[0]
return 1
return 0
