def login_session(self):
if len(g.server_host) == 0 or g.server_port == 0:
return False
try:
start_time = time.time()
magic = "P"
pack_type = 1
upload_data_head = struct.pack("<cBB8sIHII", magic, g.protocol_version, pack_type, str(self.session_id),
g.config.block_max_size, g.config.send_delay, g.config.windows_size,
g.config.windows_ack)
upload_data_head += struct.pack("<H", len(g.config.login_account)) + str(g.config.login_account)
upload_data_head += struct.pack("<H", len(g.config.login_password)) + str(g.config.login_password)
upload_post_data = encrypt_data(upload_data_head)
http_client = HTTP_client((g.server_host, g.server_port), g.proxy, g.config.use_https,
g.config.conn_life, cert=g.cert)
content, status, heads = http_client.request(method="POST", path="data", data=upload_post_data,
timeout=g.config.roundtrip_timeout)
time_cost = time.time() - start_time
if status != 200:
g.last_api_error = "session server login fail:%r" % status
xlog.warn("login session fail, status:%r", status)
return False
if len(content) < 6:
xlog.error("login data len:%d fail", len(content))
return False
info = decrypt_data(content)
magic, protocol_version, pack_type, res, message_len = struct.unpack("<cBBBH", info[:6])
message = info[6:]
if magic != "P" or protocol_version != 1 or pack_type != 1:
xlog.error("login_session time:%d head error:%s", 1000 * time_cost, utils.str2hex(info[:6]))
return False
if res != 0:
g.last_api_error = "session server login fail, code:%d msg:%s" % (res, message)
xlog.warn("login_session time:%d fail, res:%d msg:%s", 1000 * time_cost, res, message)
return False
g.last_api_error = ""
xlog.info("login_session time:%d msg:%s", 1000 * time_cost, message)
return True
except Exception as e:
xlog.exception("login_session e:%r", e)
return False
def login_session(self):
if len(g.server_host) == 0 or g.server_port == 0:
return False
try:
start_time = time.time()
magic = "P"
pack_type = 1
upload_data_head = struct.pack("<cBB8sIHII", magic, g.protocol_version, pack_type, str(self.session_id),
g.config.block_max_size, g.config.send_delay, g.config.windows_size,
g.config.windows_ack)
upload_data_head += struct.pack("<H", len(g.config.login_account)) + str(g.config.login_account)
upload_data_head += struct.pack("<H", len(g.config.login_password)) + str(g.config.login_password)
upload_post_data = encrypt_data(upload_data_head)
http_client = HTTP_client((g.server_host, g.server_port), g.proxy, g.config.use_https,
g.config.conn_life, cert=g.cert)
content, status, heads = http_client.request(method="POST", path="data", data=upload_post_data,
timeout=g.config.roundtrip_timeout)
time_cost = time.time() - start_time
if status != 200:
g.last_api_error = "session server login fail:%r" % status
xlog.warn("login session fail, status:%r", status)
return False
if len(content) < 6:
xlog.error("login data len:%d fail", len(content))
return False
info = decrypt_data(content)
magic, protocol_version, pack_type, res, message_len = struct.unpack("<cBBBH", info[:6])
message = info[6:]
if magic != "P" or protocol_version != 1 or pack_type != 1:
xlog.error("login_session time:%d head error:%s", 1000 * time_cost, utils.str2hex(info[:6]))
return False
if res != 0:
g.last_api_error = "session server login fail, code:%d msg:%s" % (res, message)
xlog.warn("login_session time:%d fail, res:%d msg:%s", 1000 * time_cost, res, message)
return False
g.last_api_error = ""
xlog.info("login_session time:%d msg:%s", 1000 * time_cost, message)
return True
except Exception as e:
xlog.exception("login_session e:%r", e)
return False
def get_api_server_http_client():
api_server = urlparse.urlparse(g.config.api_server)
http_client = HTTP_client((api_server.hostname, api_server.port),
g.proxy,
g.config.use_https,
g.config.conn_life,
cert=g.cert)
return http_client
def normal_roundtrip_worker(self, server_address):
last_roundtrip_download_size = 0
http_client = HTTP_client(server_address,
g.proxy,
g.config.use_https,
g.config.conn_life,
cert=g.cert)
while self.running:
if self.on_road_num > g.config.concurent_thread_num * 0.8:
block = True
elif last_roundtrip_download_size > g.config.block_max_size:
block = False
elif len(self.conn_list) > 0 and self.on_road_num < 1:
# keep at least one pulling thread
block = False
elif len(self.conn_list) > 0 and time.time() - self.last_download_data_time < 120 and \
self.on_road_num < g.config.concurent_thread_num * 0.1:
# busy, have data download
block = False
else:
block = True
if block:
get_timeout = 24 * 3600
else:
get_timeout = 0
# self.transfer_list[transfer_no]["stat"] = "get local data"
upload_data, send_sn = self.upload_task_queue.get(get_timeout)
transfer_no = self.get_transfer_no()
self.transfer_list[transfer_no] = {}
self.transfer_list[transfer_no]["sn"] = send_sn
send_data_len = len(upload_data)
upload_ack_data = self.ack_pool.get()
send_ack_len = len(upload_ack_data)
magic = "P"
pack_type = 2
if self.on_road_num > g.config.concurent_thread_num * 0.8:
server_timeout = 0
else:
server_timeout = g.config.roundtrip_timeout / 2
upload_data_head = struct.pack("<cBB8sIIBIH", magic,
g.protocol_version, pack_type,
str(self.session_id), transfer_no,
send_sn, server_timeout,
send_data_len, send_ack_len)
upload_post_buf = base_container.WriteBuffer(upload_data_head)
upload_post_buf.append(upload_data)
upload_post_buf.append(upload_ack_data)
upload_post_data = str(upload_post_buf)
upload_post_data = encrypt_data(upload_post_data)
try_no = 0
while self.running:
try_no += 1
sleep_time = min(try_no, 30)
self.last_roundtrip_time = time.time()
start_time = time.time()
with self.mutex:
self.on_road_num += 1
# xlog.debug("start roundtrip transfer_no:%d send_data_len:%d ack_len:%d", transfer_no, send_data_len, send_ack_len)
try:
self.transfer_list[transfer_no]["try"] = try_no
self.transfer_list[transfer_no]["stat"] = "request"
self.transfer_list[transfer_no]["start"] = time.time()
content, status, response = http_client.request(
method="POST",
path="data",
data=upload_post_data,
timeout=g.config.roundtrip_timeout)
traffic = len(upload_post_data) + len(content) + 645
self.traffic += traffic
g.quota -= traffic
except Exception as e:
xlog.exception("request except:%r retry %d", e, try_no)
time.sleep(sleep_time)
continue
finally:
with self.mutex:
self.on_road_num -= 1
if status == 405: # session_id not exist on server
if self.running:
xlog.warn(
"server session_id not exist, start reset session")
self.reset()
return
elif status == 200:
recv_len = len(content)
if recv_len < 6:
xlog.error(
"roundtrip time:%d transfer_no:%d sn:%d send:%d status:%r retry:%d",
(time.time() - start_time) * 1000, transfer_no,
send_sn, send_data_len, len(content), status,
try_no)
continue
content = decrypt_data(content)
data = base_container.ReadBuffer(content)
magic, version, pack_type = struct.unpack(
"<cBB", data.get(3))
if magic != "P" or version != g.protocol_version:
xlog.error("get data head:%s",
utils.str2hex(content[:2]))
time.sleep(100)
break
if pack_type == 3: # error report
error_code, message_len = struct.unpack(
"<BH", data.get(3))
message = data.get(message_len)
xlog.warn("error report code:%d, msg:%s", error_code,
message)
if error_code == 1: # no quota
xlog.warn("login x_server error:no quota")
self.stop()
return
else:
xlog.error("unknown error code:%d", error_code)
return
if pack_type != 2: # normal download traffic pack
xlog.error("pack type:%d", pack_type)
time.sleep(100)
break
sn, time_cost = struct.unpack("<II", data.get(8))
xlog.debug(
"roundtrip time:%d cost:%d transfer_no:%d send_sn:%d send:%d recv_sn:%d rcv:%d status:%r",
(time.time() - start_time) * 1000,
time_cost, transfer_no, send_sn, send_data_len, sn,
len(content), status)
data_len = len(data)
if (sn > 0 and data_len == 0) or (sn == 0
and data_len > 0):
xlog.warn("get sn:%d len:%d %s", sn, data_len, data)
if sn:
self.last_download_data_time = time.time()
last_roundtrip_download_size = data_len
# xlog.debug("get sn:%d len:%d", sn, data_len)
self.download_order_queue.put(sn, data)
ack_pak = struct.pack("<Q", transfer_no)
self.ack_pool.put(ack_pak)
else:
last_roundtrip_download_size = 0
if send_data_len == 0 and data_len > g.config.block_max_size:
need_more_thread_num = int(
g.config.concurent_thread_num * 0.5 -
self.on_road_num)
if need_more_thread_num > 0:
for j in range(0, need_more_thread_num):
if self.on_road_num > g.config.concurent_thread_num * 0.5:
break
self.touch_roundtrip()
break
else:
xlog.warn(
"roundtrip time:%d transfer_no:%d send_sn:%d send:%d status:%r retry:%d",
(time.time() - start_time) * 1000, transfer_no,
send_sn, send_data_len, status, try_no)
time.sleep(sleep_time)
del self.transfer_list[transfer_no]
xlog.info("roundtrip port:%d thread exit", server_address[1])
def normal_roundtrip_worker(self, server_address):
last_roundtrip_download_size = 0
http_client = HTTP_client(server_address, g.proxy, g.config.use_https, g.config.conn_life, cert=g.cert)
while self.running:
if self.on_road_num > g.config.concurent_thread_num * 0.8:
block = True
elif last_roundtrip_download_size > g.config.block_max_size:
block = False
elif len(self.conn_list) > 0 and self.on_road_num < 1:
# keep at least one pulling thread
block = False
elif len(self.conn_list) > 0 and time.time() - self.last_download_data_time < 120 and \
self.on_road_num < g.config.concurent_thread_num * 0.1:
# busy, have data download
block = False
else:
block = True
if block:
get_timeout = 24 * 3600
else:
get_timeout = 0
# self.transfer_list[transfer_no]["stat"] = "get local data"
upload_data, send_sn = self.upload_task_queue.get(get_timeout)
transfer_no = self.get_transfer_no()
self.transfer_list[transfer_no] = {}
self.transfer_list[transfer_no]["sn"] = send_sn
send_data_len = len(upload_data)
upload_ack_data = self.ack_pool.get()
send_ack_len = len(upload_ack_data)
magic = "P"
pack_type = 2
if self.on_road_num > g.config.concurent_thread_num * 0.8:
server_timeout = 0
else:
server_timeout = g.config.roundtrip_timeout / 2
upload_data_head = struct.pack("<cBB8sIIBIH", magic, g.protocol_version, pack_type, str(self.session_id),
transfer_no,
send_sn, server_timeout, send_data_len, send_ack_len)
upload_post_buf = base_container.WriteBuffer(upload_data_head)
upload_post_buf.append(upload_data)
upload_post_buf.append(upload_ack_data)
upload_post_data = str(upload_post_buf)
upload_post_data = encrypt_data(upload_post_data)
try_no = 0
while self.running:
try_no += 1
sleep_time = min(try_no, 30)
self.last_roundtrip_time = time.time()
start_time = time.time()
with self.mutex:
self.on_road_num += 1
# xlog.debug("start roundtrip transfer_no:%d send_data_len:%d ack_len:%d", transfer_no, send_data_len, send_ack_len)
try:
self.transfer_list[transfer_no]["try"] = try_no
self.transfer_list[transfer_no]["stat"] = "request"
self.transfer_list[transfer_no]["start"] = time.time()
content, status, response = http_client.request(method="POST", path="data", data=upload_post_data,
timeout=g.config.roundtrip_timeout)
traffic = len(upload_post_data) + len(content) + 645
self.traffic += traffic
g.quota -= traffic
except Exception as e:
xlog.exception("request except:%r retry %d", e, try_no)
time.sleep(sleep_time)
continue
finally:
with self.mutex:
self.on_road_num -= 1
if status == 405: # session_id not exist on server
if self.running:
xlog.warn("server session_id not exist, start reset session")
self.reset()
return
elif status == 200:
recv_len = len(content)
if recv_len < 6:
xlog.error("roundtrip time:%d transfer_no:%d sn:%d send:%d status:%r retry:%d",
(time.time() - start_time) * 1000, transfer_no, send_sn, send_data_len, len(content),
status, try_no)
continue
content = decrypt_data(content)
data = base_container.ReadBuffer(content)
magic, version, pack_type = struct.unpack("<cBB", data.get(3))
if magic != "P" or version != g.protocol_version:
xlog.error("get data head:%s", utils.str2hex(content[:2]))
time.sleep(100)
break
if pack_type == 3: # error report
error_code, message_len = struct.unpack("<BH", data.get(3))
message = data.get(message_len)
xlog.warn("error report code:%d, msg:%s", error_code, message)
if error_code == 1: # no quota
xlog.warn("login x_server error:no quota")
self.stop()
return
else:
xlog.error("unknown error code:%d", error_code)
return
if pack_type != 2: # normal download traffic pack
xlog.error("pack type:%d", pack_type)
time.sleep(100)
break
sn, time_cost = struct.unpack("<II", data.get(8))
xlog.debug(
"roundtrip time:%d cost:%d transfer_no:%d send_sn:%d send:%d recv_sn:%d rcv:%d status:%r",
(time.time() - start_time) * 1000, time_cost, transfer_no, send_sn, send_data_len, sn,
len(content), status)
data_len = len(data)
if (sn > 0 and data_len == 0) or (sn == 0 and data_len > 0):
xlog.warn("get sn:%d len:%d %s", sn, data_len, data)
if sn:
self.last_download_data_time = time.time()
last_roundtrip_download_size = data_len
# xlog.debug("get sn:%d len:%d", sn, data_len)
self.download_order_queue.put(sn, data)
ack_pak = struct.pack("<Q", transfer_no)
self.ack_pool.put(ack_pak)
else:
last_roundtrip_download_size = 0
if send_data_len == 0 and data_len > g.config.block_max_size:
need_more_thread_num = int(g.config.concurent_thread_num * 0.5 - self.on_road_num)
if need_more_thread_num > 0:
for j in range(0, need_more_thread_num):
if self.on_road_num > g.config.concurent_thread_num * 0.5:
break
self.touch_roundtrip()
break
else:
xlog.warn("roundtrip time:%d transfer_no:%d send_sn:%d send:%d status:%r retry:%d",
(time.time() - start_time) * 1000, transfer_no, send_sn, send_data_len, status, try_no)
time.sleep(sleep_time)
del self.transfer_list[transfer_no]
xlog.info("roundtrip port:%d thread exit", server_address[1])
