def __init__(self, config):
self.config = config
self.MyHost = ""
self.ListenPort = self.config["GENERAL"]["LISTEN_PORT"]
self.sigLock = thread.allocate_lock() # For locking in the sigHandler
self.monLock = thread.allocate_lock() # For keeping the monitor thread sane
self.watchUpstream = 0
# if found, convert hashed passwords from hex to string
if self.config["NTLM_AUTH"]["LM_HASHED_PW"]:
self.config["NTLM_AUTH"]["LM_HASHED_PW"] = utils.hex2str(self.config["NTLM_AUTH"]["LM_HASHED_PW"])
if self.config["NTLM_AUTH"]["NT_HASHED_PW"]:
self.config["NTLM_AUTH"]["NT_HASHED_PW"] = utils.hex2str(self.config["NTLM_AUTH"]["NT_HASHED_PW"])
if not self.config["NTLM_AUTH"]["NTLM_TO_BASIC"]:
if (self.config["NTLM_AUTH"]["LM_PART"] and not self.config["NTLM_AUTH"]["LM_HASHED_PW"]) or (
self.config["NTLM_AUTH"]["NT_PART"] and not self.config["NTLM_AUTH"]["NT_HASHED_PW"]
):
if not self.config["NTLM_AUTH"]["PASSWORD"]:
tries = 3
print "------------------------"
while tries and (not self.config["NTLM_AUTH"]["PASSWORD"]):
tries = tries - 1
self.config["NTLM_AUTH"]["PASSWORD"] = getpass.getpass("Your NT password to be used:")
if not self.config["NTLM_AUTH"]["PASSWORD"]:
print "Sorry. PASSWORD is required, bye."
sys.exit(1)
else:
# TODO: migrate this properly so placeholders aren't required
self.config["NTLM_AUTH"]["USER"] = "******"
self.config["NTLM_AUTH"]["PASSWORD"] = "******"
# compute hashed passwords if necessary
if self.config["NTLM_AUTH"]["LM_PART"] and not self.config["NTLM_AUTH"]["LM_HASHED_PW"]:
self.config["NTLM_AUTH"]["LM_HASHED_PW"] = ntlm_procs.create_LM_hashed_password(
self.config["NTLM_AUTH"]["PASSWORD"]
)
if self.config["NTLM_AUTH"]["NT_PART"] and not self.config["NTLM_AUTH"]["NT_HASHED_PW"]:
self.config["NTLM_AUTH"]["NT_HASHED_PW"] = ntlm_procs.create_NT_hashed_password(
self.config["NTLM_AUTH"]["PASSWORD"]
)
# if requested. compute and print out password hashes and exit
if self.config["NTLM_AUTH"]["PRINT_PW_HASH_AND_EXIT"]:
if self.config["NTLM_AUTH"]["LM_PART"]:
print "LM_HASHED_PW:%s" % utils.str2hex(self.config["NTLM_AUTH"]["LM_HASHED_PW"])
if self.config["NTLM_AUTH"]["NT_PART"]:
print "NT_HASHED_PW:%s" % utils.str2hex(self.config["NTLM_AUTH"]["NT_HASHED_PW"])
sys.exit(0)
def handle(self):
self.__class__.handle_num += 1
if self.try_redirect():
return
sockets = [self.conn]
try:
r, w, e = select.select(sockets, [], [])
socks_version = self.conn.recv(1, socket.MSG_PEEK)
if not socks_version:
return
if socks_version == b"\x04":
self.socks4_handler()
elif socks_version == b"\x05":
self.socks5_handler()
elif socks_version == b"C":
self.https_handler()
elif socks_version in [b"G", b"P", b"D", b"O", b"H", b"T"]:
self.http_handler()
else:
xlog.warn("socks version:%s[%s] not supported", socks_version,
utils.str2hex(socks_version))
return
except socket.error as e:
xlog.warn('socks handler read error:%r', e)
except Exception as e:
xlog.exception("any err:%r", e)
def handle(self):
try:
# xlog.debug('Connected from %r', self.client_address)
socks_version = self.read_bytes(1)
if socks_version == "\x04":
self.socks4_handler()
elif socks_version == "\x05":
self.socks5_handler()
elif socks_version == "C":
self.https_handler()
elif socks_version in ["G", "P"]:
xlog.warn(
"http proxy protocal is not supported now, please use Socks5."
)
return
else:
xlog.warn("socks version:%s not supported",
utils.str2hex(socks_version))
return
except socket.error as e:
xlog.warn('socks handler read error %r', e)
except Exception as e:
xlog.exception("any err:%r", e)
def socks5_handler(self):
sock = self.connection
auth_mode_num = ord(self.read_bytes(1))
data = self.read_bytes(auth_mode_num)
sock.send(b"\x05\x00") # socks version 5, no auth needed.
try:
data = self.read_bytes(4)
except Exception as e:
xlog.debug("socks5 auth num:%d, list:%s", auth_mode_num, utils.str2hex(data))
xlog.warn("socks5 protocol error:%r", e)
return
socks_version = ord(data[0])
if socks_version != 5:
xlog.warn("request version:%d error", socks_version)
return
command = ord(data[1])
if command != 1: # 1. Tcp connect
xlog.warn("request not supported command mode:%d", command)
sock.send(b"\x05\x07\x00\x01") # Command not supported
return
addrtype_pack = data[3]
addrtype = ord(addrtype_pack)
if addrtype == 1: # IPv4
addr_pack = self.read_bytes(4)
addr = socket.inet_ntoa(addr_pack)
elif addrtype == 3: # Domain name
domain_len_pack = self.read_bytes(1)[0]
domain_len = ord(domain_len_pack)
domain = self.read_bytes(domain_len)
addr_pack = domain_len_pack + domain
addr = domain
elif addrtype == 4: # IPv6
addr_pack = self.read_bytes(16)
addr = socket.inet_ntop(socket.AF_INET6, addr_pack)
else:
xlog.warn("request address type unknown:%d", addrtype)
sock.send(b"\x05\x07\x00\x01") # Command not supported
return
port = struct.unpack('>H', self.rfile.read(2))[0]
conn_id = proxy_session.create_conn(sock, addr, port)
if not conn_id:
xlog.warn("create conn fail")
reply = b"\x05\x01\x00" + addrtype_pack + addr_pack + struct.pack(">H", port)
sock.send(reply)
return
xlog.info("socks5 %r connect to %s:%d conn_id:%d", self.client_address, addr, port, conn_id)
reply = b"\x05\x00\x00" + addrtype_pack + addr_pack + struct.pack(">H", port)
sock.send(reply)
if len(self.read_buffer) - self.buffer_start:
g.session.conn_list[conn_id].transfer_received_data(self.read_buffer[self.buffer_start:])
g.session.conn_list[conn_id].start(block=True)
def handle(self):
self.__class__.handle_num += 1
try:
r, w, e = select.select([self.connection], [], [])
socks_version = self.read_bytes(1)
if not socks_version:
return
if socks_version == "\x04":
self.socks4_handler()
elif socks_version == "\x05":
self.socks5_handler()
elif socks_version == "C":
self.https_handler()
elif socks_version in ["G", "P", "D", "O", "H", "T"]:
self.http_handler(socks_version)
return
else:
xlog.warn("socks version:%s not supported",
utils.str2hex(socks_version))
return
except socket.error as e:
xlog.debug('socks handler read error %r', e)
return
except Exception as e:
xlog.exception("any err:%r", e)
def socks5_handler(self):
sock = self.connection
auth_mode_num = ord(self.read_bytes(1))
data = self.read_bytes(auth_mode_num)
sock.send(b"\x05\x00") # socks version 5, no auth needed.
try:
data = self.read_bytes(4)
except Exception as e:
xlog.debug("socks5 auth num:%d, list:%s", auth_mode_num, utils.str2hex(data))
xlog.exception("socks5 protocol error:%r", e)
return
socks_version = ord(data[0])
if socks_version != 5:
xlog.warn("request version:%d error", socks_version)
return
command = ord(data[1])
if command != 1: # 1. Tcp connect
xlog.warn("request not supported command mode:%d", command)
sock.send(b"\x05\x07\x00\x01") # Command not supported
return
addrtype_pack = data[3]
addrtype = ord(addrtype_pack)
if addrtype == 1: # IPv4
addr_pack = self.read_bytes(4)
addr = socket.inet_ntoa(addr_pack)
elif addrtype == 3: # Domain name
domain_len_pack = self.read_bytes(1)[0]
domain_len = ord(domain_len_pack)
domain = self.read_bytes(domain_len)
addr_pack = domain_len_pack + domain
addr = domain
elif addrtype == 4: # IPv6
addr_pack = self.read_bytes(16)
addr = socket.inet_ntop(socket.AF_INET6, addr_pack)
else:
xlog.warn("request address type unknown:%d", addrtype)
sock.send(b"\x05\x07\x00\x01") # Command not supported
return
port = struct.unpack(">H", self.rfile.read(2))[0]
conn_id = g.session.create_conn(sock, addr, port)
if not conn_id:
xlog.warn("create conn fail")
reply = b"\x05\x01\x00" + addrtype_pack + addr_pack + struct.pack(">H", port)
sock.send(reply)
return
xlog.info("socks5 %r connect to %s:%d conn_id:%d", self.client_address, addr, port, conn_id)
reply = b"\x05\x00\x00" + addrtype_pack + addr_pack + struct.pack(">H", port)
sock.send(reply)
if len(self.read_buffer) - self.buffer_start:
g.session.conn_list[conn_id].transfer_received_data(self.read_buffer[self.buffer_start :])
g.session.conn_list[conn_id].start(block=True)
def unknown_part(bin_str):
""
res = ''
res = res + 'Hex : %s\n' % utils.str2hex(bin_str, ' ')
res = res + 'String : %s\n' % utils.str2prn_str(bin_str, ' ')
res = res + 'Decimal: %s\n' % utils.str2dec(bin_str, ' ')
return res
def unknown_part(bin_str):
""
res = ''
res = res + 'Hex : %s\n' % utils.str2hex(bin_str, ' ')
res = res + 'String : %s\n' % utils.str2prn_str(bin_str, ' ')
res = res + 'Decimal: %s\n' % utils.str2dec(bin_str, ' ')
return res
def item(item_str):
""
item = {}
res = ''
item['len1'] = utils.bytes2int(item_str[0:2])
item['len2'] = utils.bytes2int(item_str[2:4])
item['offset'] = utils.bytes2int(item_str[4:6])
res = res + '%s\n\nlength (two times), offset, delimiter\n' % (utils.str2hex(item_str))
res = res + '%s decimal: %3d # length 1\n' % (utils.int2hex_str(item['len1']), item['len1'])
res = res + '%s decimal: %3d # length 2\n' % (utils.int2hex_str(item['len2']), item['len2'])
res = res + '%s decimal: %3d # offset\n' % (utils.int2hex_str(item['offset']), item['offset'])
res = res + '%s # delimiter (two zeros)\n\n' % utils.str2hex(item_str[-2:])
item['string'] = res
return item
def socks5_handler(self):
sock = self.conn
socks_version = ord(self.read_bytes(1))
auth_mode_num = ord(self.read_bytes(1))
data = self.read_bytes(auth_mode_num)
sock.send(b"\x05\x00") # socks version 5, no auth needed.
try:
data = self.read_bytes(4)
except Exception as e:
xlog.debug("socks5 auth num:%d, list:%s", auth_mode_num,
utils.str2hex(data))
xlog.warn("socks5 protocol error:%r", e)
return
socks_version = ord(data[0:1])
if socks_version != 5:
xlog.warn("request version:%d error", socks_version)
return
command = ord(data[1:2])
if command != 1: # 1. Tcp connect
xlog.warn("request not supported command mode:%d", command)
sock.send(b"\x05\x07\x00\x01") # Command not supported
return
addrtype_pack = data[3:4]
addrtype = ord(addrtype_pack)
if addrtype == 1: # IPv4
addr_pack = self.read_bytes(4)
addr = socket.inet_ntoa(addr_pack)
elif addrtype == 3: # Domain name
domain_len_pack = self.read_bytes(1)[0:1]
domain_len = ord(domain_len_pack)
domain = self.read_bytes(domain_len)
addr_pack = domain_len_pack + domain
addr = domain
elif addrtype == 4: # IPv6
addr_pack = self.read_bytes(16)
addr = socket.inet_ntop(socket.AF_INET6, addr_pack)
else:
xlog.warn("request address type unknown:%d", addrtype)
sock.send(b"\x05\x07\x00\x01") # Command not supported
return
port = struct.unpack('>H', self.rfile.read(2))[0]
# xlog.debug("socks5 %r connect to %s:%d", self.client_address, addr, port)
reply = b"\x05\x00\x00" + addrtype_pack + addr_pack + struct.pack(
">H", port)
sock.send(reply)
if addrtype in [1, 4]:
handle_ip_proxy(sock, addr, port, self.client_address)
else:
handle_domain_proxy(sock, addr, port, self.client_address)
def socks5_handler(self):
sock = self.conn
socks_version = ord(self.read_bytes(1))
auth_mode_num = ord(self.read_bytes(1))
data = self.read_bytes(auth_mode_num)
sock.send(b"\x05\x00") # socks version 5, no auth needed.
try:
data = self.read_bytes(4)
except Exception as e:
xlog.debug("socks5 auth num:%d, list:%s", auth_mode_num, utils.str2hex(data))
xlog.warn("socks5 protocol error:%r", e)
return
socks_version = ord(data[0])
if socks_version != 5:
xlog.warn("request version:%d error", socks_version)
return
command = ord(data[1])
if command != 1: # 1. Tcp connect
xlog.warn("request not supported command mode:%d", command)
sock.send(b"\x05\x07\x00\x01") # Command not supported
return
addrtype_pack = data[3]
addrtype = ord(addrtype_pack)
if addrtype == 1: # IPv4
addr_pack = self.read_bytes(4)
addr = socket.inet_ntoa(addr_pack)
elif addrtype == 3: # Domain name
domain_len_pack = self.read_bytes(1)[0]
domain_len = ord(domain_len_pack)
domain = self.read_bytes(domain_len)
addr_pack = domain_len_pack + domain
addr = domain
elif addrtype == 4: # IPv6
addr_pack = self.read_bytes(16)
addr = socket.inet_ntop(socket.AF_INET6, addr_pack)
else:
xlog.warn("request address type unknown:%d", addrtype)
sock.send(b"\x05\x07\x00\x01") # Command not supported
return
port = struct.unpack('>H', self.rfile.read(2))[0]
# xlog.debug("socks5 %r connect to %s:%d", self.client_address, addr, port)
reply = b"\x05\x00\x00" + addrtype_pack + addr_pack + struct.pack(">H", port)
sock.send(reply)
if addrtype in [1, 4]:
handle_ip_proxy(sock, addr, port, self.client_address)
else:
handle_domain_proxy(sock, addr, port, self.client_address)
def item(item_str):
""
item = {}
res = ''
item['len1'] = utils.bytes2int(item_str[0:2])
item['len2'] = utils.bytes2int(item_str[2:4])
item['offset'] = utils.bytes2int(item_str[4:6])
res = res + '%s\n\nlength (two times), offset, delimiter\n' % (
utils.str2hex(item_str))
res = res + '%s decimal: %3d # length 1\n' % (utils.int2hex_str(
item['len1']), item['len1'])
res = res + '%s decimal: %3d # length 2\n' % (utils.int2hex_str(
item['len2']), item['len2'])
res = res + '%s decimal: %3d # offset\n' % (utils.int2hex_str(
item['offset']), item['offset'])
res = res + '%s # delimiter (two zeros)\n\n' % utils.str2hex(
item_str[-2:])
item['string'] = res
return item
def login_session(self):
if len(g.server_host) == 0 or g.server_port == 0:
return False
try:
start_time = time.time()
magic = "P"
pack_type = 1
upload_data_head = struct.pack("<cBB8sIHII", magic, g.protocol_version, pack_type, str(self.session_id),
g.config.block_max_size, g.config.send_delay, g.config.windows_size,
g.config.windows_ack)
upload_data_head += struct.pack("<H", len(g.config.login_account)) + str(g.config.login_account)
upload_data_head += struct.pack("<H", len(g.config.login_password)) + str(g.config.login_password)
upload_post_data = encrypt_data(upload_data_head)
http_client = HTTP_client((g.server_host, g.server_port), g.proxy, g.config.use_https,
g.config.conn_life, cert=g.cert)
content, status, heads = http_client.request(method="POST", path="data", data=upload_post_data,
timeout=g.config.roundtrip_timeout)
time_cost = time.time() - start_time
if status != 200:
g.last_api_error = "session server login fail:%r" % status
xlog.warn("login session fail, status:%r", status)
return False
if len(content) < 6:
xlog.error("login data len:%d fail", len(content))
return False
info = decrypt_data(content)
magic, protocol_version, pack_type, res, message_len = struct.unpack("<cBBBH", info[:6])
message = info[6:]
if magic != "P" or protocol_version != 1 or pack_type != 1:
xlog.error("login_session time:%d head error:%s", 1000 * time_cost, utils.str2hex(info[:6]))
return False
if res != 0:
g.last_api_error = "session server login fail, code:%d msg:%s" % (res, message)
xlog.warn("login_session time:%d fail, res:%d msg:%s", 1000 * time_cost, res, message)
return False
g.last_api_error = ""
xlog.info("login_session time:%d msg:%s", 1000 * time_cost, message)
return True
except Exception as e:
xlog.exception("login_session e:%r", e)
return False
def login_session(self):
if len(g.server_host) == 0 or g.server_port == 0:
return False
try:
start_time = time.time()
magic = "P"
pack_type = 1
upload_data_head = struct.pack("<cBB8sIHII", magic, g.protocol_version, pack_type, str(self.session_id),
g.config.block_max_size, g.config.send_delay, g.config.windows_size,
g.config.windows_ack)
upload_data_head += struct.pack("<H", len(g.config.login_account)) + str(g.config.login_account)
upload_data_head += struct.pack("<H", len(g.config.login_password)) + str(g.config.login_password)
upload_post_data = encrypt_data(upload_data_head)
http_client = HTTP_client((g.server_host, g.server_port), g.proxy, g.config.use_https,
g.config.conn_life, cert=g.cert)
content, status, heads = http_client.request(method="POST", path="data", data=upload_post_data,
timeout=g.config.roundtrip_timeout)
time_cost = time.time() - start_time
if status != 200:
g.last_api_error = "session server login fail:%r" % status
xlog.warn("login session fail, status:%r", status)
return False
if len(content) < 6:
xlog.error("login data len:%d fail", len(content))
return False
info = decrypt_data(content)
magic, protocol_version, pack_type, res, message_len = struct.unpack("<cBBBH", info[:6])
message = info[6:]
if magic != "P" or protocol_version != 1 or pack_type != 1:
xlog.error("login_session time:%d head error:%s", 1000 * time_cost, utils.str2hex(info[:6]))
return False
if res != 0:
g.last_api_error = "session server login fail, code:%d msg:%s" % (res, message)
xlog.warn("login_session time:%d fail, res:%d msg:%s", 1000 * time_cost, res, message)
return False
g.last_api_error = ""
xlog.info("login_session time:%d msg:%s", 1000 * time_cost, message)
return True
except Exception as e:
xlog.exception("login_session e:%r", e)
return False
def flags(flag_str):
""
res = ''
res = res + '%s\n\n' % utils.str2hex(flag_str)
flags = utils.bytes2int(flag_str[0:2])
res = res + '%s # flags\n' % (utils.int2hex_str(flags))
res = res + 'Binary:\nlayout 87654321 87654321\n'
res = res + ' %s %s\n' % (utils.byte2bin_str(flag_str[1]), utils.byte2bin_str(flag_str[0]))
flags2 = utils.bytes2int(flag_str[2:4])
res = res + '%s # more flags ???\n' % (utils.int2hex_str(flags2))
res = res + 'Binary:\nlayout 87654321 87654321\n'
res = res + ' %s %s\n' % (utils.byte2bin_str(flag_str[3]), utils.byte2bin_str(flag_str[2]))
#res = res + '%s # delimiter ???\n' % m_hex[(cur + 2) * 2: (cur + 4) * 2]
return res
def handle(self):
try:
# xlog.debug('Connected from %r', self.client_address)
socks_version = self.read_bytes(1)
if socks_version == "\x04":
self.socks4_handler()
elif socks_version == "\x05":
self.socks5_handler()
elif socks_version == "C":
self.https_handler()
else:
xlog.warn("socks version:%s not supported", utils.str2hex(socks_version))
return
except socket.error as e:
xlog.exception("socks handler read error %r", e)
except Exception as e:
xlog.exception("any err:%r", e)
def flags(flag_str):
""
res = ''
res = res + '%s\n\n' % utils.str2hex(flag_str)
flags = utils.bytes2int(flag_str[0:2])
res = res + '%s # flags\n' % (utils.int2hex_str(flags))
res = res + 'Binary:\nlayout 87654321 87654321\n'
res = res + ' %s %s\n' % (utils.byte2bin_str(
flag_str[1]), utils.byte2bin_str(flag_str[0]))
flags2 = utils.bytes2int(flag_str[2:4])
res = res + '%s # more flags ???\n' % (
utils.int2hex_str(flags2))
res = res + 'Binary:\nlayout 87654321 87654321\n'
res = res + ' %s %s\n' % (utils.byte2bin_str(
flag_str[3]), utils.byte2bin_str(flag_str[2]))
#res = res + '%s # delimiter ???\n' % m_hex[(cur + 2) * 2: (cur + 4) * 2]
return res
def handle(self):
self.__class__.handle_num += 1
try:
dst = self.conn.getsockopt(socket.SOL_IP, SO_ORIGINAL_DST, 16)
dst_port, srv_ip = struct.unpack("!2xH4s8x", dst)
ip_str = socket.inet_ntoa(srv_ip)
if dst_port != g.config.proxy_port and not utils.is_private_ip(
ip_str):
xlog.debug("Redirect to:%s:%d from:%s", ip_str, dst_port,
self.client_address)
return handle_ip_proxy(self.conn, ip_str, dst_port,
self.client_address)
except:
pass
sockets = [self.conn]
try:
r, w, e = select.select(sockets, [], [])
socks_version = self.conn.recv(1, socket.MSG_PEEK)
if not socks_version:
return
if socks_version == "\x04":
self.socks4_handler()
elif socks_version == "\x05":
self.socks5_handler()
elif socks_version == "C":
self.https_handler()
elif socks_version in ["G", "P", "D", "O", "H", "T"]:
self.http_handler()
else:
xlog.warn("socks version:%s[%s] not supported", socks_version,
utils.str2hex(socks_version))
return
except socket.error as e:
xlog.warn('socks handler read error:%r', e)
except Exception as e:
xlog.exception("any err:%r", e)
def debug_message1(msg):
""
m_ = base64.decodestring(msg)
m_hex = utils.str2hex(m_)
res = ''
res = res + '==============================================================\n'
res = res + 'NTLM Message 1 report:\n'
res = res + '---------------------------------\n'
res = res + 'Base64: %s\n' % msg
res = res + 'String: %s\n' % utils.str2prn_str(m_)
res = res + 'Hex: %s\n' % m_hex
cur = 0
res = res + '---------------------------------\n'
cur_len = 12
res = res + 'Header %d/%d:\n%s\n\n' % (cur, cur_len, m_hex[0:24])
res = res + '%s\nmethod name 0/8\n%s # C string\n\n' % (
m_hex[0:16], utils.str2prn_str(m_[0:8]))
res = res + '0x%s%s # message type\n' % (m_hex[18:20],
m_hex[16:18])
res = res + '%s # delimiter (zeros)\n' % m_hex[20:24]
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = 4
res = res + 'Flags %d/%d\n' % (cur, cur_len)
res = res + flags(m_[cur:cur + cur_len])
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = len(m_) - cur
res = res + 'Rest of the message %d/%d:\n' % (cur, cur_len)
res = res + unknown_part(m_[cur:cur + cur_len])
res = res + '\nEnd of message 1 report.\n'
return res
def debug_message1(msg):
""
m_ = base64.decodestring(msg)
m_hex = utils.str2hex(m_)
res = ''
res = res + '==============================================================\n'
res = res + 'NTLM Message 1 report:\n'
res = res + '---------------------------------\n'
res = res + 'Base64: %s\n' % msg
res = res + 'String: %s\n' % utils.str2prn_str(m_)
res = res + 'Hex: %s\n' % m_hex
cur = 0
res = res + '---------------------------------\n'
cur_len = 12
res = res + 'Header %d/%d:\n%s\n\n' % (cur, cur_len, m_hex[0:24])
res = res + '%s\nmethod name 0/8\n%s # C string\n\n' % (m_hex[0:16], utils.str2prn_str(m_[0:8]))
res = res + '0x%s%s # message type\n' % (m_hex[18:20], m_hex[16:18])
res = res + '%s # delimiter (zeros)\n' % m_hex[20:24]
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = 4
res = res + 'Flags %d/%d\n' % (cur, cur_len)
res = res + flags(m_[cur: cur + cur_len])
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = len(m_) - cur
res = res + 'Rest of the message %d/%d:\n' % (cur, cur_len)
res = res + unknown_part(m_[cur: cur + cur_len])
res = res + '\nEnd of message 1 report.\n'
return res
def handle(self):
self.__class__.handle_num += 1
try:
dst = self.conn.getsockopt(socket.SOL_IP, SO_ORIGINAL_DST, 16)
dst_port, srv_ip = struct.unpack("!2xH4s8x", dst)
ip_str = socket.inet_ntoa(srv_ip)
if dst_port != g.config.proxy_port and not utils.is_private_ip(ip_str):
xlog.debug("Redirect to:%s:%d from:%s", ip_str, dst_port, self.client_address)
return handle_ip_proxy(self.conn, ip_str, dst_port, self.client_address)
except:
pass
sockets = [self.conn]
try:
r, w, e = select.select(sockets, [], [])
socks_version = self.conn.recv(1, socket.MSG_PEEK)
if not socks_version:
return
if socks_version == "\x04":
self.socks4_handler()
elif socks_version == "\x05":
self.socks5_handler()
elif socks_version == "C":
self.https_handler()
elif socks_version in ["G", "P", "D", "O", "H", "T"]:
self.http_handler()
else:
xlog.warn("socks version:%s[%s] not supported", socks_version, utils.str2hex(socks_version))
return
except socket.error as e:
xlog.warn('socks handler read error:%r', e)
except Exception as e:
xlog.exception("any err:%r", e)
def debug_message2(msg):
""
m_ = base64.decodestring(msg)
m_hex = utils.str2hex(m_)
res = ''
res = res + '==============================================================\n'
res = res + 'NTLM Message 2 report:\n'
res = res + '---------------------------------\n'
res = res + 'Base64: %s\n' % msg
res = res + 'String: %s\n' % utils.str2prn_str(m_)
res = res + 'Hex: %s\n' % m_hex
cur = 0
res = res + '---------------------------------\n'
cur_len = 12
res = res + 'Header %d/%d:\n%s\n\n' % (cur, cur_len, m_hex[0:24])
res = res + '%s\nmethod name 0/8\n%s # C string\n\n' % (m_hex[0:16], utils.str2prn_str(m_[0:8]))
res = res + '0x%s%s # message type\n' % (m_hex[18:20], m_hex[16:18])
res = res + '%s # delimiter (zeros)\n' % m_hex[20:24]
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = 8
res = res + 'Lengths and Positions %d/%d\n%s\n\n' % (cur, cur_len, m_hex[cur * 2 :(cur + cur_len) * 2])
cur_len = 8
res = res + 'Domain ??? %d/%d\n' % (cur, cur_len)
dom = item(m_[cur:cur+cur_len])
res = res + dom['string']
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = 4
res = res + 'Flags %d/%d\n' % (cur, cur_len)
res = res + flags(m_[cur: cur + cur_len])
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = 8
res = res + 'NONCE %d/%d\n%s\n\n' % (cur, cur_len, m_hex[cur * 2 :(cur + cur_len) * 2])
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = dom['offset'] - cur
res = res + 'Unknown data %d/%d:\n' % (cur, cur_len)
res = res + unknown_part(m_[cur: cur + cur_len])
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = dom['len1']
res = res + 'Domain ??? %d/%d:\n' % (cur, cur_len)
res = res + 'Hex: %s\n' % m_hex[cur * 2: (cur + cur_len) * 2]
res = res + 'String: %s\n\n' % utils.str2prn_str(m_[cur : cur + cur_len])
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = len(m_) - cur
res = res + 'Rest of the message %d/%d:\n' % (cur, cur_len)
res = res + unknown_part(m_[cur: cur + cur_len])
res = res + '\nEnd of message 2 report.\n'
return res
def debug_message2(msg):
""
m_ = base64.decodestring(msg)
m_hex = utils.str2hex(m_)
res = ''
res = res + '==============================================================\n'
res = res + 'NTLM Message 2 report:\n'
res = res + '---------------------------------\n'
res = res + 'Base64: %s\n' % msg
res = res + 'String: %s\n' % utils.str2prn_str(m_)
res = res + 'Hex: %s\n' % m_hex
cur = 0
res = res + '---------------------------------\n'
cur_len = 12
res = res + 'Header %d/%d:\n%s\n\n' % (cur, cur_len, m_hex[0:24])
res = res + '%s\nmethod name 0/8\n%s # C string\n\n' % (
m_hex[0:16], utils.str2prn_str(m_[0:8]))
res = res + '0x%s%s # message type\n' % (m_hex[18:20],
m_hex[16:18])
res = res + '%s # delimiter (zeros)\n' % m_hex[20:24]
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = 8
res = res + 'Lengths and Positions %d/%d\n%s\n\n' % (
cur, cur_len, m_hex[cur * 2:(cur + cur_len) * 2])
cur_len = 8
res = res + 'Domain ??? %d/%d\n' % (cur, cur_len)
dom = item(m_[cur:cur + cur_len])
res = res + dom['string']
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = 4
res = res + 'Flags %d/%d\n' % (cur, cur_len)
res = res + flags(m_[cur:cur + cur_len])
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = 8
res = res + 'NONCE %d/%d\n%s\n\n' % (cur, cur_len,
m_hex[cur * 2:(cur + cur_len) * 2])
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = dom['offset'] - cur
res = res + 'Unknown data %d/%d:\n' % (cur, cur_len)
res = res + unknown_part(m_[cur:cur + cur_len])
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = dom['len1']
res = res + 'Domain ??? %d/%d:\n' % (cur, cur_len)
res = res + 'Hex: %s\n' % m_hex[cur * 2:(cur + cur_len) * 2]
res = res + 'String: %s\n\n' % utils.str2prn_str(m_[cur:cur + cur_len])
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = len(m_) - cur
res = res + 'Rest of the message %d/%d:\n' % (cur, cur_len)
res = res + unknown_part(m_[cur:cur + cur_len])
res = res + '\nEnd of message 2 report.\n'
return res
def login_session(self):
if len(g.server_host) == 0:
return False
start_time = time.time()
while time.time() - start_time < 30:
try:
start_time = time.time()
magic = "P"
pack_type = 1
upload_data_head = struct.pack(
"<cBB8sIHIIHH", magic, g.protocol_version, pack_type,
str(self.session_id), g.config.max_payload,
g.config.send_delay, g.config.windows_size,
g.config.windows_ack, g.config.resend_timeout,
g.config.ack_delay)
upload_data_head += struct.pack(
"<H", len(g.config.login_account)) + str(
g.config.login_account)
upload_data_head += struct.pack(
"<H", len(g.config.login_password)) + str(
g.config.login_password)
upload_post_data = encrypt_data(upload_data_head)
content, status, response = g.http_client.request(
method="POST",
host=g.server_host,
path="/data",
data=upload_post_data,
timeout=g.config.network_timeout)
time_cost = time.time() - start_time
if status == 521:
g.last_api_error = "session server is down."
xlog.warn(
"login session server is down, try get new server.")
g.server_host = None
return False
if status != 200:
g.last_api_error = "session server login fail:%r" % status
xlog.warn("login session fail, status:%r", status)
continue
if len(content) < 6:
g.last_api_error = "session server protocol fail, login res len:%d" % len(
content)
xlog.error("login data len:%d fail", len(content))
continue
info = decrypt_data(content)
magic, protocol_version, pack_type, res, message_len = struct.unpack(
"<cBBBH", info[:6])
message = info[6:]
if magic != "P" or protocol_version != g.protocol_version or pack_type != 1:
xlog.error("login_session time:%d head error:%s",
1000 * time_cost, utils.str2hex(info[:6]))
return False
if res != 0:
g.last_api_error = "session server login fail, code:%d msg:%s" % (
res, message)
xlog.warn("login_session time:%d fail, res:%d msg:%s",
1000 * time_cost, res, message)
return False
g.last_api_error = ""
xlog.info("login_session %s time:%d msg:%s", self.session_id,
1000 * time_cost, message)
return True
except Exception as e:
xlog.exception("login_session e:%r", e)
time.sleep(1)
return False
def normal_roundtrip_worker(self):
while self.running:
data, ack = self.get_send_data()
if not self.running:
return
send_data_len = len(data)
send_ack_len = len(ack)
transfer_no = self.get_transfer_no()
magic = "P"
pack_type = 2
if self.on_road_num > g.config.concurent_thread_num * 0.6:
server_timeout = 0
else:
server_timeout = g.config.roundtrip_timeout
request_session_id = self.session_id
upload_data_head = struct.pack("<cBB8sIBIH", magic,
g.protocol_version, pack_type,
str(self.session_id), transfer_no,
server_timeout, send_data_len,
send_ack_len)
upload_post_buf = base_container.WriteBuffer(upload_data_head)
upload_post_buf.append(data)
upload_post_buf.append(ack)
upload_post_data = str(upload_post_buf)
upload_post_data = encrypt_data(upload_post_data)
self.last_send_time = time.time()
sleep_time = 1
start_time = time.time()
with self.lock:
self.on_road_num += 1
self.transfer_list[transfer_no] = {}
self.transfer_list[transfer_no]["stat"] = "request"
self.transfer_list[transfer_no]["start"] = start_time
#xlog.debug("start roundtrip transfer_no:%d send_data_len:%d ack_len:%d timeout:%d",
# transfer_no, send_data_len, send_ack_len, server_timeout)
try:
content, status, response = g.http_client.request(
method="POST",
host=g.server_host,
path="/data?tid=%d" % transfer_no,
data=upload_post_data,
headers={"Content-Length": str(len(upload_post_data))},
timeout=server_timeout + g.config.network_timeout)
traffic = len(upload_post_data) + len(content) + 645
self.traffic += traffic
g.quota -= traffic
except Exception as e:
xlog.exception("request except:%r ", e)
time.sleep(sleep_time)
continue
finally:
with self.lock:
self.on_road_num -= 1
try:
if transfer_no in self.transfer_list:
del self.transfer_list[transfer_no]
except:
pass
g.stat["roundtrip_num"] += 1
roundtrip_time = (time.time() - start_time) * 1000
if status == 521:
xlog.warn("X-tunnel server is down, try get new server.")
g.server_host = None
self.stop()
login_process()
return
if status != 200:
xlog.warn(
"roundtrip time:%d transfer_no:%d send:%d status:%r ",
roundtrip_time, transfer_no, send_data_len, status)
time.sleep(sleep_time)
continue
recv_len = len(content)
if recv_len < 6:
xlog.warn(
"roundtrip time:%d transfer_no:%d send:%d recv:%d Head",
roundtrip_time, transfer_no, send_data_len, recv_len)
continue
content = decrypt_data(content)
payload = base_container.ReadBuffer(content)
magic, version, pack_type = struct.unpack("<cBB", payload.get(3))
if magic != "P" or version != g.protocol_version:
xlog.warn("get data head:%s", utils.str2hex(content[:2]))
time.sleep(sleep_time)
continue
if pack_type == 3: # error report
error_code, message_len = struct.unpack("<BH", payload.get(3))
message = payload.get(message_len)
# xlog.warn("report code:%d, msg:%s", error_code, message)
if error_code == 1:
# no quota
xlog.warn("x_server error:no quota")
self.stop()
return
elif error_code == 2:
# unpack error
xlog.warn(
"roundtrip time:%d transfer_no:%d send:%d recv:%d unpack_error:%s",
roundtrip_time, transfer_no, send_data_len,
len(content), message)
continue
elif error_code == 3:
# session not exist
if self.session_id == request_session_id:
xlog.warn(
"server session_id:%s not exist, reset session.",
request_session_id)
self.reset()
return
else:
continue
else:
xlog.error("unknown error code:%d, message:%s", error_code,
message)
time.sleep(sleep_time)
continue
if pack_type != 2: # normal download traffic pack
xlog.error("pack type:%d", pack_type)
time.sleep(100)
continue
time_cost, server_send_pool_size, data_len, ack_len = struct.unpack(
"<IIIH", payload.get(14))
xlog.debug(
"roundtrip time:%d cost:%d transfer_no:%d send:%d rcv:%d ",
roundtrip_time, time_cost, transfer_no, send_data_len,
len(content))
self.trigger_more(server_send_pool_size)
rtt = roundtrip_time - time_cost
rtt = max(100, rtt)
speed = (send_data_len + len(content) + 400) / rtt
response.worker.update_debug_data(rtt, send_data_len, len(content),
speed)
if rtt > 8000:
xlog.debug("rtt:%d speed:%d trace:%s", rtt, speed,
response.worker.get_trace())
xlog.debug("task trace:%s", response.task.get_trace())
g.stat["slow_roundtrip"] += 1
try:
data = payload.get_buf(data_len)
ack = payload.get_buf(ack_len)
except:
xlog.debug("data not enough")
continue
try:
self.roundtrip_process(data, ack)
self.last_receive_time = time.time()
except Exception as e:
xlog.exception("data process:%r", e)
continue
xlog.info("roundtrip thread exit")
def debug_message3(msg):
""
m_ = base64.decodestring(msg)
m_hex = utils.str2hex(m_)
res = ''
res = res + '==============================================================\n'
res = res + 'NTLM Message 3 report:\n'
res = res + '---------------------------------\n'
res = res + 'Base64: %s\n' % msg
res = res + 'String: %s\n' % utils.str2prn_str(m_)
res = res + 'Hex: %s\n' % m_hex
cur = 0
res = res + '---------------------------------\n'
cur_len = 12
res = res + 'Header %d/%d:\n%s\n\n' % (cur, cur_len, m_hex[0:24])
res = res + '%s\nmethod name 0/8\n%s # C string\n\n' % (
m_hex[0:16], utils.str2prn_str(m_[0:8]))
res = res + '0x%s%s # message type\n' % (m_hex[18:20],
m_hex[16:18])
res = res + '%s # delimiter (zeros)\n' % m_hex[20:24]
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = 48
res = res + 'Lengths and Positions %d/%d\n%s\n\n' % (
cur, cur_len, m_hex[cur * 2:(cur + cur_len) * 2])
cur_len = 8
res = res + 'LAN Manager response %d/%d\n' % (cur, cur_len)
lmr = item(m_[cur:cur + cur_len])
res = res + lmr['string']
cur = cur + cur_len
cur_len = 8
res = res + 'NT response %d/%d\n' % (cur, cur_len)
ntr = item(m_[cur:cur + cur_len])
res = res + ntr['string']
cur = cur + cur_len
cur_len = 8
res = res + 'Domain string %d/%d\n' % (cur, cur_len)
dom = item(m_[cur:cur + cur_len])
res = res + dom['string']
cur = cur + cur_len
cur_len = 8
res = res + 'User string %d/%d\n' % (cur, cur_len)
user = item(m_[cur:cur + cur_len])
res = res + user['string']
cur = cur + cur_len
cur_len = 8
res = res + 'Host string %d/%d\n' % (cur, cur_len)
host = item(m_[cur:cur + cur_len])
res = res + host['string']
cur = cur + cur_len
cur_len = 8
res = res + 'Unknow item record %d/%d\n' % (cur, cur_len)
unknown = item(m_[cur:cur + cur_len])
res = res + unknown['string']
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = 4
res = res + 'Flags %d/%d\n' % (cur, cur_len)
res = res + flags(m_[cur:cur + cur_len])
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = dom['len1'] + user['len1'] + host['len1']
res = res + 'Domain, User, Host strings %d/%d\n%s\n%s\n\n' % (
cur, cur_len, m_hex[cur * 2:(cur + cur_len) * 2],
utils.str2prn_str(m_[cur:cur + cur_len]))
cur_len = dom['len1']
res = res + '%s\n' % m_hex[cur * 2:(cur + cur_len) * 2]
res = res + 'Domain name %d/%d:\n' % (cur, cur_len)
res = res + '%s\n\n' % (utils.str2prn_str(m_[cur:(cur + cur_len)]))
cur = cur + cur_len
cur_len = user['len1']
res = res + '%s\n' % m_hex[cur * 2:(cur + cur_len) * 2]
res = res + 'User name %d/%d:\n' % (cur, cur_len)
res = res + '%s\n\n' % (utils.str2prn_str(m_[cur:(cur + cur_len)]))
cur = cur + cur_len
cur_len = host['len1']
res = res + '%s\n' % m_hex[cur * 2:(cur + cur_len) * 2]
res = res + 'Host name %d/%d:\n' % (cur, cur_len)
res = res + '%s\n\n' % (utils.str2prn_str(m_[cur:(cur + cur_len)]))
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = lmr['len1']
res = res + 'LAN Manager response %d/%d\n%s\n\n' % (
cur, cur_len, m_hex[cur * 2:(cur + cur_len) * 2])
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = ntr['len1']
res = res + 'NT response %d/%d\n%s\n\n' % (
cur, cur_len, m_hex[cur * 2:(cur + cur_len) * 2])
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = len(m_) - cur
res = res + 'Rest of the message %d/%d:\n' % (cur, cur_len)
res = res + unknown_part(m_[cur:cur + cur_len])
res = res + '\nEnd of message 3 report.\n'
return res
def normal_roundtrip_worker(self):
while self.running:
data, ack = self.get_send_data()
if not self.running:
return
send_data_len = len(data)
send_ack_len = len(ack)
transfer_no = self.get_transfer_no()
magic = "P"
pack_type = 2
if self.on_road_num > g.config.concurent_thread_num * 0.6:
server_timeout = 0
else:
server_timeout = g.config.roundtrip_timeout
request_session_id = self.session_id
upload_data_head = struct.pack("<cBB8sIBIH", magic, g.protocol_version, pack_type,
str(self.session_id), transfer_no,
server_timeout, send_data_len, send_ack_len)
upload_post_buf = base_container.WriteBuffer(upload_data_head)
upload_post_buf.append(data)
upload_post_buf.append(ack)
upload_post_data = str(upload_post_buf)
upload_post_data = encrypt_data(upload_post_data)
self.last_send_time = time.time()
sleep_time = 1
start_time = time.time()
with self.lock:
self.on_road_num += 1
self.transfer_list[transfer_no] = {}
self.transfer_list[transfer_no]["stat"] = "request"
self.transfer_list[transfer_no]["start"] = start_time
#xlog.debug("start roundtrip transfer_no:%d send_data_len:%d ack_len:%d timeout:%d",
# transfer_no, send_data_len, send_ack_len, server_timeout)
try:
content, status, response = g.http_client.request(method="POST", host=g.server_host,
path="/data", data=upload_post_data,
headers={"Content-Length": str(len(upload_post_data))},
timeout=server_timeout + g.config.network_timeout)
traffic = len(upload_post_data) + len(content) + 645
self.traffic += traffic
g.quota -= traffic
except Exception as e:
xlog.exception("request except:%r ", e)
time.sleep(sleep_time)
continue
finally:
with self.lock:
self.on_road_num -= 1
try:
if transfer_no in self.transfer_list:
del self.transfer_list[transfer_no]
except:
pass
g.stat["roundtrip_num"] += 1
roundtrip_time = (time.time() - start_time) * 1000
if status == 521:
xlog.warn("X-tunnel server is down, try get new server.")
request_balance(update_server=True)
self.reset()
return
if status != 200:
xlog.warn("roundtrip time:%d transfer_no:%d send:%d status:%r ",
roundtrip_time, transfer_no, send_data_len, status)
time.sleep(sleep_time)
continue
recv_len = len(content)
if recv_len < 6:
xlog.warn("roundtrip time:%d transfer_no:%d send:%d recv:%d Head",
roundtrip_time, transfer_no, send_data_len, recv_len)
continue
content = decrypt_data(content)
payload = base_container.ReadBuffer(content)
magic, version, pack_type = struct.unpack("<cBB", payload.get(3))
if magic != "P" or version != g.protocol_version:
xlog.warn("get data head:%s", utils.str2hex(content[:2]))
time.sleep(sleep_time)
continue
if pack_type == 3: # error report
error_code, message_len = struct.unpack("<BH", payload.get(3))
message = payload.get(message_len)
# xlog.warn("report code:%d, msg:%s", error_code, message)
if error_code == 1:
# no quota
xlog.warn("x_server error:no quota")
self.stop()
return
elif error_code == 2:
# unpack error
xlog.warn("roundtrip time:%d transfer_no:%d send:%d recv:%d unpack_error:%s",
roundtrip_time, transfer_no, send_data_len, len(content), message)
continue
elif error_code == 3:
# session not exist
if self.session_id == request_session_id:
xlog.warn("server session_id:%s not exist, reset session.", request_session_id)
self.reset()
return
else:
continue
else:
xlog.error("unknown error code:%d, message:%s", error_code, message)
time.sleep(sleep_time)
continue
if pack_type != 2: # normal download traffic pack
xlog.error("pack type:%d", pack_type)
time.sleep(100)
continue
time_cost, server_send_pool_size, data_len, ack_len = struct.unpack("<IIIH", payload.get(14))
xlog.debug("roundtrip time:%d cost:%d transfer_no:%d send:%d rcv:%d ",
roundtrip_time, time_cost, transfer_no, send_data_len, len(content))
self.trigger_more(server_send_pool_size)
rtt = roundtrip_time - time_cost
rtt = max(100, rtt)
speed = (send_data_len + len(content) + 400) / rtt
response.worker.update_debug_data(rtt, send_data_len, len(content), speed)
if rtt > 8000:
xlog.debug("rtt:%d speed:%d trace:%s", rtt, speed, response.worker.get_trace())
xlog.debug("task trace:%s", response.task.get_trace())
g.stat["slow_roundtrip"] += 1
try:
data = payload.get_buf(data_len)
ack = payload.get_buf(ack_len)
except:
xlog.debug("data not enough")
continue
try:
self.roundtrip_process(data, ack)
self.last_receive_time = time.time()
except Exception as e:
xlog.exception("data process:%r", e)
continue
xlog.info("roundtrip thread exit")
def login_session(self):
if len(g.server_host) == 0:
return False
start_time = time.time()
while time.time() - start_time < 30:
try:
start_time = time.time()
magic = "P"
pack_type = 1
upload_data_head = struct.pack("<cBB8sIHIIHH", magic, g.protocol_version, pack_type, str(self.session_id),
g.config.max_payload, g.config.send_delay, g.config.windows_size,
g.config.windows_ack, g.config.resend_timeout, g.config.ack_delay)
upload_data_head += struct.pack("<H", len(g.config.login_account)) + str(g.config.login_account)
upload_data_head += struct.pack("<H", len(g.config.login_password)) + str(g.config.login_password)
upload_post_data = encrypt_data(upload_data_head)
content, status, response = g.http_client.request(method="POST", host=g.server_host, path="/data",
data=upload_post_data,
timeout=g.config.network_timeout)
time_cost = time.time() - start_time
if status == 521:
g.last_api_error = "session server is down."
xlog.warn("login session server is down, try get new server.")
request_balance(update_server=True)
continue
if status != 200:
g.last_api_error = "session server login fail:%r" % status
xlog.warn("login session fail, status:%r", status)
continue
if len(content) < 6:
g.last_api_error = "session server protocol fail, login res len:%d" % len(content)
xlog.error("login data len:%d fail", len(content))
continue
info = decrypt_data(content)
magic, protocol_version, pack_type, res, message_len = struct.unpack("<cBBBH", info[:6])
message = info[6:]
if magic != "P" or protocol_version != g.protocol_version or pack_type != 1:
xlog.error("login_session time:%d head error:%s", 1000 * time_cost, utils.str2hex(info[:6]))
return False
if res != 0:
g.last_api_error = "session server login fail, code:%d msg:%s" % (res, message)
xlog.warn("login_session time:%d fail, res:%d msg:%s", 1000 * time_cost, res, message)
return False
g.last_api_error = ""
xlog.info("login_session %s time:%d msg:%s", self.session_id, 1000 * time_cost, message)
return True
except Exception as e:
xlog.exception("login_session e:%r", e)
time.sleep(1)
return False
def debug_message3(msg):
""
m_ = base64.decodestring(msg)
m_hex = utils.str2hex(m_)
res = ''
res = res + '==============================================================\n'
res = res + 'NTLM Message 3 report:\n'
res = res + '---------------------------------\n'
res = res + 'Base64: %s\n' % msg
res = res + 'String: %s\n' % utils.str2prn_str(m_)
res = res + 'Hex: %s\n' % m_hex
cur = 0
res = res + '---------------------------------\n'
cur_len = 12
res = res + 'Header %d/%d:\n%s\n\n' % (cur, cur_len, m_hex[0:24])
res = res + '%s\nmethod name 0/8\n%s # C string\n\n' % (m_hex[0:16], utils.str2prn_str(m_[0:8]))
res = res + '0x%s%s # message type\n' % (m_hex[18:20], m_hex[16:18])
res = res + '%s # delimiter (zeros)\n' % m_hex[20:24]
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = 48
res = res + 'Lengths and Positions %d/%d\n%s\n\n' % (cur, cur_len, m_hex[cur * 2 :(cur + cur_len) * 2])
cur_len = 8
res = res + 'LAN Manager response %d/%d\n' % (cur, cur_len)
lmr = item(m_[cur:cur+cur_len])
res = res + lmr['string']
cur = cur + cur_len
cur_len = 8
res = res + 'NT response %d/%d\n' % (cur, cur_len)
ntr = item(m_[cur:cur+cur_len])
res = res + ntr['string']
cur = cur + cur_len
cur_len = 8
res = res + 'Domain string %d/%d\n' % (cur, cur_len)
dom = item(m_[cur:cur+cur_len])
res = res + dom['string']
cur = cur + cur_len
cur_len = 8
res = res + 'User string %d/%d\n' % (cur, cur_len)
user = item(m_[cur:cur+cur_len])
res = res + user['string']
cur = cur + cur_len
cur_len = 8
res = res + 'Host string %d/%d\n' % (cur, cur_len)
host = item(m_[cur:cur+cur_len])
res = res + host['string']
cur = cur + cur_len
cur_len = 8
res = res + 'Unknow item record %d/%d\n' % (cur, cur_len)
unknown = item(m_[cur:cur+cur_len])
res = res + unknown['string']
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = 4
res = res + 'Flags %d/%d\n' % (cur, cur_len)
res = res + flags(m_[cur: cur + cur_len])
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = dom['len1'] + user['len1'] + host['len1']
res = res + 'Domain, User, Host strings %d/%d\n%s\n%s\n\n' % (cur, cur_len, m_hex[cur * 2 :(cur + cur_len) * 2], utils.str2prn_str(m_[cur:cur + cur_len]))
cur_len = dom['len1']
res = res + '%s\n' % m_hex[cur * 2: (cur + cur_len) * 2]
res = res + 'Domain name %d/%d:\n' % (cur, cur_len)
res = res + '%s\n\n' % (utils.str2prn_str(m_[cur: (cur + cur_len)]))
cur = cur + cur_len
cur_len = user['len1']
res = res + '%s\n' % m_hex[cur * 2: (cur + cur_len) * 2]
res = res + 'User name %d/%d:\n' % (cur, cur_len)
res = res + '%s\n\n' % (utils.str2prn_str(m_[cur: (cur + cur_len)]))
cur = cur + cur_len
cur_len = host['len1']
res = res + '%s\n' % m_hex[cur * 2: (cur + cur_len) * 2]
res = res + 'Host name %d/%d:\n' % (cur, cur_len)
res = res + '%s\n\n' % (utils.str2prn_str(m_[cur: (cur + cur_len)]))
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = lmr['len1']
res = res + 'LAN Manager response %d/%d\n%s\n\n' % (cur, cur_len, m_hex[cur * 2 :(cur + cur_len) * 2])
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = ntr['len1']
res = res + 'NT response %d/%d\n%s\n\n' % (cur, cur_len, m_hex[cur * 2 :(cur + cur_len) * 2])
cur = cur + cur_len
res = res + '---------------------------------\n'
cur_len = len(m_) - cur
res = res + 'Rest of the message %d/%d:\n' % (cur, cur_len)
res = res + unknown_part(m_[cur: cur + cur_len])
res = res + '\nEnd of message 3 report.\n'
return res
def normal_roundtrip_worker(self, server_address):
last_roundtrip_download_size = 0
while self.running:
if self.on_road_num > g.config.concurent_thread_num * 0.9:
block = True
xlog.debug("block for too many on road")
elif last_roundtrip_download_size > 0:
block = False
elif len(self.conn_list) > 0 and self.on_road_num < 1:
# keep at least one pulling thread
block = False
elif len(self.conn_list) > 0 and time.time() - self.last_download_data_time < 120 and \
self.on_road_num < g.config.concurent_thread_num * 0.1:
# busy, have data download
block = False
else:
block = True
# xlog.debug("block for none")
if block:
get_timeout = 24 * 3600
else:
get_timeout = 0
# self.transfer_list[transfer_no]["stat"] = "get local data"
upload_data, send_sn = self.upload_task_queue.get(get_timeout)
transfer_no = self.get_transfer_no()
self.transfer_list[transfer_no] = {}
self.transfer_list[transfer_no]["sn"] = send_sn
send_data_len = len(upload_data)
upload_ack_data = self.ack_pool.get()
send_ack_len = len(upload_ack_data)
magic = "P"
pack_type = 2
if self.on_road_num > g.config.concurent_thread_num * 0.8:
server_timeout = 0
else:
if last_roundtrip_download_size >= g.config.block_max_size:
server_timeout = 1
else:
server_timeout = g.config.roundtrip_timeout / 2
request_session_id = self.session_id
upload_data_head = struct.pack("<cBB8sIIBIH", magic, g.protocol_version, pack_type, str(self.session_id),
transfer_no,
send_sn, server_timeout, send_data_len, send_ack_len)
upload_post_buf = base_container.WriteBuffer(upload_data_head)
upload_post_buf.append(upload_data)
upload_post_buf.append(upload_ack_data)
upload_post_data = str(upload_post_buf)
upload_post_data = encrypt_data(upload_post_data)
try_no = 0
while self.running:
try_no += 1
sleep_time = min(try_no, 30)
self.last_roundtrip_time = time.time()
start_time = time.time()
with self.mutex:
self.on_road_num += 1
# xlog.debug("start roundtrip transfer_no:%d send_data_len:%d ack_len:%d", transfer_no, send_data_len, send_ack_len)
try:
self.transfer_list[transfer_no]["try"] = try_no
self.transfer_list[transfer_no]["stat"] = "request"
self.transfer_list[transfer_no]["start"] = time.time()
content, status, response = g.http_client.request(method="POST", host=g.server_host,
path="/data", data=upload_post_data,
timeout=server_timeout + g.config.network_timeout)
traffic = len(upload_post_data) + len(content) + 645
self.traffic += traffic
g.quota -= traffic
except Exception as e:
xlog.exception("request except:%r retry %d", e, try_no)
time.sleep(sleep_time)
if transfer_no not in self.transfer_list:
break
else:
continue
finally:
with self.mutex:
self.on_road_num -= 1
g.stat["roundtrip_num"] += 1
if status == 521:
xlog.warn("X-tunnel server is down, try get new server.")
request_balance(update_server=True)
self.reset()
return
if status == 405: # session_id not exist on server
if self.running:
if self.session_id == request_session_id:
xlog.warn("server session_id:%s not exist, reset session.", request_session_id)
self.reset()
return
elif status == 200:
recv_len = len(content)
if recv_len < 6:
xlog.warn("roundtrip time:%d transfer_no:%d sn:%d send:%d len:%d status:%r retry:%d",
(time.time() - start_time) * 1000, transfer_no, send_sn, send_data_len, len(content),
status, try_no)
continue
content = decrypt_data(content)
data = base_container.ReadBuffer(content)
magic, version, pack_type = struct.unpack("<cBB", data.get(3))
if magic != "P" or version != g.protocol_version:
xlog.error("get data head:%s", utils.str2hex(content[:2]))
time.sleep(100)
break
if pack_type == 3: # error report
error_code, message_len = struct.unpack("<BH", data.get(3))
message = data.get(message_len)
xlog.warn("error report code:%d, msg:%s", error_code, message)
if error_code == 1: # no quota
xlog.warn("login x_server error:no quota")
self.stop()
return
else:
xlog.error("unknown error code:%d", error_code)
return
if pack_type != 2: # normal download traffic pack
xlog.error("pack type:%d", pack_type)
time.sleep(100)
break
sn, time_cost = struct.unpack("<II", data.get(8))
roundtrip_time = (time.time() - start_time) * 1000
xlog.debug(
"roundtrip time:%d cost:%d transfer_no:%d send_sn:%d send:%d recv_sn:%d rcv:%d status:%r",
roundtrip_time, time_cost, transfer_no, send_sn, send_data_len, sn,
len(content), status)
rtt = roundtrip_time - time_cost
speed = (send_data_len + len(content) + 400) / rtt
response.worker.update_debug_data(rtt, send_data_len, len(content), speed)
if rtt > 8000:
xlog.warn("rtt:%d speed:%d trace:%s", rtt, speed, response.worker.get_trace())
xlog.warn("task trace:%s", response.task.get_trace())
g.stat["slow_roundtrip"] += 1
data_len = len(data)
if (sn > 0 and data_len == 0) or (sn == 0 and data_len > 0):
xlog.warn("get sn:%d len:%d %s", sn, data_len, data)
if send_data_len == 0 and \
data_len >= g.config.block_max_size and \
last_roundtrip_download_size >= g.config.block_max_size:
if self.on_road_num < int(g.config.concurent_thread_num * 0.8):
self.touch_roundtrip()
if sn:
self.last_download_data_time = time.time()
last_roundtrip_download_size = data_len
# xlog.debug("get sn:%d len:%d", sn, data_len)
# self.download_order_queue.put(sn, data)
try:
self.download_data_processor(data)
except Exception as e:
xlog.warn("data process:%r", e)
continue
ack_pak = struct.pack("<Q", transfer_no)
self.ack_pool.put(ack_pak)
else:
last_roundtrip_download_size = 0
break
else:
g.stat["timeout_roundtrip"] += 1
xlog.warn("roundtrip time:%d transfer_no:%d send_sn:%d send:%d status:%r retry:%d",
(time.time() - start_time) * 1000, transfer_no, send_sn, send_data_len, status, try_no)
time.sleep(sleep_time)
try:
if transfer_no in self.transfer_list:
del self.transfer_list[transfer_no]
except:
pass
xlog.info("roundtrip port:%d thread exit", server_address[1])
def normal_roundtrip_worker(self, server_address):
last_roundtrip_download_size = 0
while self.running:
if self.on_road_num > g.config.concurent_thread_num * 0.8:
block = True
elif last_roundtrip_download_size > g.config.block_max_size:
block = False
elif len(self.conn_list) > 0 and self.on_road_num < 1:
# keep at least one pulling thread
block = False
elif len(self.conn_list) > 0 and time.time() - self.last_download_data_time < 120 and \
self.on_road_num < g.config.concurent_thread_num * 0.1:
# busy, have data download
block = False
else:
block = True
if block:
get_timeout = 24 * 3600
else:
get_timeout = 0
# self.transfer_list[transfer_no]["stat"] = "get local data"
upload_data, send_sn = self.upload_task_queue.get(get_timeout)
transfer_no = self.get_transfer_no()
self.transfer_list[transfer_no] = {}
self.transfer_list[transfer_no]["sn"] = send_sn
send_data_len = len(upload_data)
upload_ack_data = self.ack_pool.get()
send_ack_len = len(upload_ack_data)
magic = "P"
pack_type = 2
if self.on_road_num > g.config.concurent_thread_num * 0.8:
server_timeout = 0
else:
server_timeout = g.config.roundtrip_timeout / 2
request_session_id = self.session_id
upload_data_head = struct.pack("<cBB8sIIBIH", magic, g.protocol_version, pack_type, str(self.session_id),
transfer_no,
send_sn, server_timeout, send_data_len, send_ack_len)
upload_post_buf = base_container.WriteBuffer(upload_data_head)
upload_post_buf.append(upload_data)
upload_post_buf.append(upload_ack_data)
upload_post_data = str(upload_post_buf)
upload_post_data = encrypt_data(upload_post_data)
try_no = 0
while self.running:
try_no += 1
sleep_time = min(try_no, 30)
self.last_roundtrip_time = time.time()
start_time = time.time()
with self.mutex:
self.on_road_num += 1
# xlog.debug("start roundtrip transfer_no:%d send_data_len:%d ack_len:%d", transfer_no, send_data_len, send_ack_len)
try:
self.transfer_list[transfer_no]["try"] = try_no
self.transfer_list[transfer_no]["stat"] = "request"
self.transfer_list[transfer_no]["start"] = time.time()
content, status, response = g.http_client.request(method="POST", host=g.server_host,
path="/data", data=upload_post_data,
timeout=g.config.roundtrip_timeout)
traffic = len(upload_post_data) + len(content) + 645
self.traffic += traffic
g.quota -= traffic
except Exception as e:
xlog.exception("request except:%r retry %d", e, try_no)
time.sleep(sleep_time)
if transfer_no not in self.transfer_list:
break
else:
continue
finally:
with self.mutex:
self.on_road_num -= 1
if status == 405: # session_id not exist on server
if self.running:
if self.session_id == request_session_id:
xlog.warn("server session_id:%s not exist, reset session.", request_session_id)
self.reset()
return
elif status == 200:
recv_len = len(content)
if recv_len < 6:
xlog.error("roundtrip time:%d transfer_no:%d sn:%d send:%d len:%d status:%r retry:%d",
(time.time() - start_time) * 1000, transfer_no, send_sn, send_data_len, len(content),
status, try_no)
continue
content = decrypt_data(content)
data = base_container.ReadBuffer(content)
magic, version, pack_type = struct.unpack("<cBB", data.get(3))
if magic != "P" or version != g.protocol_version:
xlog.error("get data head:%s", utils.str2hex(content[:2]))
time.sleep(100)
break
if pack_type == 3: # error report
error_code, message_len = struct.unpack("<BH", data.get(3))
message = data.get(message_len)
xlog.warn("error report code:%d, msg:%s", error_code, message)
if error_code == 1: # no quota
xlog.warn("login x_server error:no quota")
self.stop()
return
else:
xlog.error("unknown error code:%d", error_code)
return
if pack_type != 2: # normal download traffic pack
xlog.error("pack type:%d", pack_type)
time.sleep(100)
break
sn, time_cost = struct.unpack("<II", data.get(8))
xlog.debug(
"roundtrip time:%d cost:%d transfer_no:%d send_sn:%d send:%d recv_sn:%d rcv:%d status:%r",
(time.time() - start_time) * 1000, time_cost, transfer_no, send_sn, send_data_len, sn,
len(content), status)
data_len = len(data)
if (sn > 0 and data_len == 0) or (sn == 0 and data_len > 0):
xlog.warn("get sn:%d len:%d %s", sn, data_len, data)
if sn:
self.last_download_data_time = time.time()
last_roundtrip_download_size = data_len
# xlog.debug("get sn:%d len:%d", sn, data_len)
self.download_order_queue.put(sn, data)
ack_pak = struct.pack("<Q", transfer_no)
self.ack_pool.put(ack_pak)
else:
last_roundtrip_download_size = 0
if send_data_len == 0 and data_len > g.config.block_max_size:
need_more_thread_num = int(g.config.concurent_thread_num * 0.5 - self.on_road_num)
if need_more_thread_num > 0:
for j in range(0, need_more_thread_num):
if self.on_road_num > g.config.concurent_thread_num * 0.5:
break
self.touch_roundtrip()
break
else:
xlog.warn("roundtrip time:%d transfer_no:%d send_sn:%d send:%d status:%r retry:%d",
(time.time() - start_time) * 1000, transfer_no, send_sn, send_data_len, status, try_no)
time.sleep(sleep_time)
try:
if transfer_no in self.transfer_list:
del self.transfer_list[transfer_no]
except:
pass
xlog.info("roundtrip port:%d thread exit", server_address[1])
def normal_roundtrip_worker(self, server_address):
last_roundtrip_download_size = 0
http_client = HTTP_client(server_address, g.proxy, g.config.use_https, g.config.conn_life, cert=g.cert)
while self.running:
if self.on_road_num > g.config.concurent_thread_num * 0.8:
block = True
elif last_roundtrip_download_size > g.config.block_max_size:
block = False
elif len(self.conn_list) > 0 and self.on_road_num < 1:
# keep at least one pulling thread
block = False
elif len(self.conn_list) > 0 and time.time() - self.last_download_data_time < 120 and \
self.on_road_num < g.config.concurent_thread_num * 0.1:
# busy, have data download
block = False
else:
block = True
if block:
get_timeout = 24 * 3600
else:
get_timeout = 0
# self.transfer_list[transfer_no]["stat"] = "get local data"
upload_data, send_sn = self.upload_task_queue.get(get_timeout)
transfer_no = self.get_transfer_no()
self.transfer_list[transfer_no] = {}
self.transfer_list[transfer_no]["sn"] = send_sn
send_data_len = len(upload_data)
upload_ack_data = self.ack_pool.get()
send_ack_len = len(upload_ack_data)
magic = "P"
pack_type = 2
if self.on_road_num > g.config.concurent_thread_num * 0.8:
server_timeout = 0
else:
server_timeout = g.config.roundtrip_timeout / 2
upload_data_head = struct.pack("<cBB8sIIBIH", magic, g.protocol_version, pack_type, str(self.session_id),
transfer_no,
send_sn, server_timeout, send_data_len, send_ack_len)
upload_post_buf = base_container.WriteBuffer(upload_data_head)
upload_post_buf.append(upload_data)
upload_post_buf.append(upload_ack_data)
upload_post_data = str(upload_post_buf)
upload_post_data = encrypt_data(upload_post_data)
try_no = 0
while self.running:
try_no += 1
sleep_time = min(try_no, 30)
self.last_roundtrip_time = time.time()
start_time = time.time()
with self.mutex:
self.on_road_num += 1
# xlog.debug("start roundtrip transfer_no:%d send_data_len:%d ack_len:%d", transfer_no, send_data_len, send_ack_len)
try:
self.transfer_list[transfer_no]["try"] = try_no
self.transfer_list[transfer_no]["stat"] = "request"
self.transfer_list[transfer_no]["start"] = time.time()
content, status, response = http_client.request(method="POST", path="data", data=upload_post_data,
timeout=g.config.roundtrip_timeout)
traffic = len(upload_post_data) + len(content) + 645
self.traffic += traffic
g.quota -= traffic
except Exception as e:
xlog.exception("request except:%r retry %d", e, try_no)
time.sleep(sleep_time)
continue
finally:
with self.mutex:
self.on_road_num -= 1
if status == 405: # session_id not exist on server
if self.running:
xlog.warn("server session_id not exist, start reset session")
self.reset()
return
elif status == 200:
recv_len = len(content)
if recv_len < 6:
xlog.error("roundtrip time:%d transfer_no:%d sn:%d send:%d status:%r retry:%d",
(time.time() - start_time) * 1000, transfer_no, send_sn, send_data_len, len(content),
status, try_no)
continue
content = decrypt_data(content)
data = base_container.ReadBuffer(content)
magic, version, pack_type = struct.unpack("<cBB", data.get(3))
if magic != "P" or version != g.protocol_version:
xlog.error("get data head:%s", utils.str2hex(content[:2]))
time.sleep(100)
break
if pack_type == 3: # error report
error_code, message_len = struct.unpack("<BH", data.get(3))
message = data.get(message_len)
xlog.warn("error report code:%d, msg:%s", error_code, message)
if error_code == 1: # no quota
xlog.warn("login x_server error:no quota")
self.stop()
return
else:
xlog.error("unknown error code:%d", error_code)
return
if pack_type != 2: # normal download traffic pack
xlog.error("pack type:%d", pack_type)
time.sleep(100)
break
sn, time_cost = struct.unpack("<II", data.get(8))
xlog.debug(
"roundtrip time:%d cost:%d transfer_no:%d send_sn:%d send:%d recv_sn:%d rcv:%d status:%r",
(time.time() - start_time) * 1000, time_cost, transfer_no, send_sn, send_data_len, sn,
len(content), status)
data_len = len(data)
if (sn > 0 and data_len == 0) or (sn == 0 and data_len > 0):
xlog.warn("get sn:%d len:%d %s", sn, data_len, data)
if sn:
self.last_download_data_time = time.time()
last_roundtrip_download_size = data_len
# xlog.debug("get sn:%d len:%d", sn, data_len)
self.download_order_queue.put(sn, data)
ack_pak = struct.pack("<Q", transfer_no)
self.ack_pool.put(ack_pak)
else:
last_roundtrip_download_size = 0
if send_data_len == 0 and data_len > g.config.block_max_size:
need_more_thread_num = int(g.config.concurent_thread_num * 0.5 - self.on_road_num)
if need_more_thread_num > 0:
for j in range(0, need_more_thread_num):
if self.on_road_num > g.config.concurent_thread_num * 0.5:
break
self.touch_roundtrip()
break
else:
xlog.warn("roundtrip time:%d transfer_no:%d send_sn:%d send:%d status:%r retry:%d",
(time.time() - start_time) * 1000, transfer_no, send_sn, send_data_len, status, try_no)
time.sleep(sleep_time)
del self.transfer_list[transfer_no]
xlog.info("roundtrip port:%d thread exit", server_address[1])
