def __init__(self, config): self.config = config self.MyHost = "" self.ListenPort = self.config["GENERAL"]["LISTEN_PORT"] self.sigLock = thread.allocate_lock() # For locking in the sigHandler self.monLock = thread.allocate_lock() # For keeping the monitor thread sane self.watchUpstream = 0 # if found, convert hashed passwords from hex to string if self.config["NTLM_AUTH"]["LM_HASHED_PW"]: self.config["NTLM_AUTH"]["LM_HASHED_PW"] = utils.hex2str(self.config["NTLM_AUTH"]["LM_HASHED_PW"]) if self.config["NTLM_AUTH"]["NT_HASHED_PW"]: self.config["NTLM_AUTH"]["NT_HASHED_PW"] = utils.hex2str(self.config["NTLM_AUTH"]["NT_HASHED_PW"]) if not self.config["NTLM_AUTH"]["NTLM_TO_BASIC"]: if (self.config["NTLM_AUTH"]["LM_PART"] and not self.config["NTLM_AUTH"]["LM_HASHED_PW"]) or ( self.config["NTLM_AUTH"]["NT_PART"] and not self.config["NTLM_AUTH"]["NT_HASHED_PW"] ): if not self.config["NTLM_AUTH"]["PASSWORD"]: tries = 3 print "------------------------" while tries and (not self.config["NTLM_AUTH"]["PASSWORD"]): tries = tries - 1 self.config["NTLM_AUTH"]["PASSWORD"] = getpass.getpass("Your NT password to be used:") if not self.config["NTLM_AUTH"]["PASSWORD"]: print "Sorry. PASSWORD is required, bye." sys.exit(1) else: # TODO: migrate this properly so placeholders aren't required self.config["NTLM_AUTH"]["USER"] = "******" self.config["NTLM_AUTH"]["PASSWORD"] = "******" # compute hashed passwords if necessary if self.config["NTLM_AUTH"]["LM_PART"] and not self.config["NTLM_AUTH"]["LM_HASHED_PW"]: self.config["NTLM_AUTH"]["LM_HASHED_PW"] = ntlm_procs.create_LM_hashed_password( self.config["NTLM_AUTH"]["PASSWORD"] ) if self.config["NTLM_AUTH"]["NT_PART"] and not self.config["NTLM_AUTH"]["NT_HASHED_PW"]: self.config["NTLM_AUTH"]["NT_HASHED_PW"] = ntlm_procs.create_NT_hashed_password( self.config["NTLM_AUTH"]["PASSWORD"] ) # if requested. compute and print out password hashes and exit if self.config["NTLM_AUTH"]["PRINT_PW_HASH_AND_EXIT"]: if self.config["NTLM_AUTH"]["LM_PART"]: print "LM_HASHED_PW:%s" % utils.str2hex(self.config["NTLM_AUTH"]["LM_HASHED_PW"]) if self.config["NTLM_AUTH"]["NT_PART"]: print "NT_HASHED_PW:%s" % utils.str2hex(self.config["NTLM_AUTH"]["NT_HASHED_PW"]) sys.exit(0)
def handle(self): self.__class__.handle_num += 1 if self.try_redirect(): return sockets = [self.conn] try: r, w, e = select.select(sockets, [], []) socks_version = self.conn.recv(1, socket.MSG_PEEK) if not socks_version: return if socks_version == b"\x04": self.socks4_handler() elif socks_version == b"\x05": self.socks5_handler() elif socks_version == b"C": self.https_handler() elif socks_version in [b"G", b"P", b"D", b"O", b"H", b"T"]: self.http_handler() else: xlog.warn("socks version:%s[%s] not supported", socks_version, utils.str2hex(socks_version)) return except socket.error as e: xlog.warn('socks handler read error:%r', e) except Exception as e: xlog.exception("any err:%r", e)
def handle(self): try: # xlog.debug('Connected from %r', self.client_address) socks_version = self.read_bytes(1) if socks_version == "\x04": self.socks4_handler() elif socks_version == "\x05": self.socks5_handler() elif socks_version == "C": self.https_handler() elif socks_version in ["G", "P"]: xlog.warn( "http proxy protocal is not supported now, please use Socks5." ) return else: xlog.warn("socks version:%s not supported", utils.str2hex(socks_version)) return except socket.error as e: xlog.warn('socks handler read error %r', e) except Exception as e: xlog.exception("any err:%r", e)
def socks5_handler(self): sock = self.connection auth_mode_num = ord(self.read_bytes(1)) data = self.read_bytes(auth_mode_num) sock.send(b"\x05\x00") # socks version 5, no auth needed. try: data = self.read_bytes(4) except Exception as e: xlog.debug("socks5 auth num:%d, list:%s", auth_mode_num, utils.str2hex(data)) xlog.warn("socks5 protocol error:%r", e) return socks_version = ord(data[0]) if socks_version != 5: xlog.warn("request version:%d error", socks_version) return command = ord(data[1]) if command != 1: # 1. Tcp connect xlog.warn("request not supported command mode:%d", command) sock.send(b"\x05\x07\x00\x01") # Command not supported return addrtype_pack = data[3] addrtype = ord(addrtype_pack) if addrtype == 1: # IPv4 addr_pack = self.read_bytes(4) addr = socket.inet_ntoa(addr_pack) elif addrtype == 3: # Domain name domain_len_pack = self.read_bytes(1)[0] domain_len = ord(domain_len_pack) domain = self.read_bytes(domain_len) addr_pack = domain_len_pack + domain addr = domain elif addrtype == 4: # IPv6 addr_pack = self.read_bytes(16) addr = socket.inet_ntop(socket.AF_INET6, addr_pack) else: xlog.warn("request address type unknown:%d", addrtype) sock.send(b"\x05\x07\x00\x01") # Command not supported return port = struct.unpack('>H', self.rfile.read(2))[0] conn_id = proxy_session.create_conn(sock, addr, port) if not conn_id: xlog.warn("create conn fail") reply = b"\x05\x01\x00" + addrtype_pack + addr_pack + struct.pack(">H", port) sock.send(reply) return xlog.info("socks5 %r connect to %s:%d conn_id:%d", self.client_address, addr, port, conn_id) reply = b"\x05\x00\x00" + addrtype_pack + addr_pack + struct.pack(">H", port) sock.send(reply) if len(self.read_buffer) - self.buffer_start: g.session.conn_list[conn_id].transfer_received_data(self.read_buffer[self.buffer_start:]) g.session.conn_list[conn_id].start(block=True)
def handle(self): self.__class__.handle_num += 1 try: r, w, e = select.select([self.connection], [], []) socks_version = self.read_bytes(1) if not socks_version: return if socks_version == "\x04": self.socks4_handler() elif socks_version == "\x05": self.socks5_handler() elif socks_version == "C": self.https_handler() elif socks_version in ["G", "P", "D", "O", "H", "T"]: self.http_handler(socks_version) return else: xlog.warn("socks version:%s not supported", utils.str2hex(socks_version)) return except socket.error as e: xlog.debug('socks handler read error %r', e) return except Exception as e: xlog.exception("any err:%r", e)
def socks5_handler(self): sock = self.connection auth_mode_num = ord(self.read_bytes(1)) data = self.read_bytes(auth_mode_num) sock.send(b"\x05\x00") # socks version 5, no auth needed. try: data = self.read_bytes(4) except Exception as e: xlog.debug("socks5 auth num:%d, list:%s", auth_mode_num, utils.str2hex(data)) xlog.exception("socks5 protocol error:%r", e) return socks_version = ord(data[0]) if socks_version != 5: xlog.warn("request version:%d error", socks_version) return command = ord(data[1]) if command != 1: # 1. Tcp connect xlog.warn("request not supported command mode:%d", command) sock.send(b"\x05\x07\x00\x01") # Command not supported return addrtype_pack = data[3] addrtype = ord(addrtype_pack) if addrtype == 1: # IPv4 addr_pack = self.read_bytes(4) addr = socket.inet_ntoa(addr_pack) elif addrtype == 3: # Domain name domain_len_pack = self.read_bytes(1)[0] domain_len = ord(domain_len_pack) domain = self.read_bytes(domain_len) addr_pack = domain_len_pack + domain addr = domain elif addrtype == 4: # IPv6 addr_pack = self.read_bytes(16) addr = socket.inet_ntop(socket.AF_INET6, addr_pack) else: xlog.warn("request address type unknown:%d", addrtype) sock.send(b"\x05\x07\x00\x01") # Command not supported return port = struct.unpack(">H", self.rfile.read(2))[0] conn_id = g.session.create_conn(sock, addr, port) if not conn_id: xlog.warn("create conn fail") reply = b"\x05\x01\x00" + addrtype_pack + addr_pack + struct.pack(">H", port) sock.send(reply) return xlog.info("socks5 %r connect to %s:%d conn_id:%d", self.client_address, addr, port, conn_id) reply = b"\x05\x00\x00" + addrtype_pack + addr_pack + struct.pack(">H", port) sock.send(reply) if len(self.read_buffer) - self.buffer_start: g.session.conn_list[conn_id].transfer_received_data(self.read_buffer[self.buffer_start :]) g.session.conn_list[conn_id].start(block=True)
def unknown_part(bin_str): "" res = '' res = res + 'Hex : %s\n' % utils.str2hex(bin_str, ' ') res = res + 'String : %s\n' % utils.str2prn_str(bin_str, ' ') res = res + 'Decimal: %s\n' % utils.str2dec(bin_str, ' ') return res
def item(item_str): "" item = {} res = '' item['len1'] = utils.bytes2int(item_str[0:2]) item['len2'] = utils.bytes2int(item_str[2:4]) item['offset'] = utils.bytes2int(item_str[4:6]) res = res + '%s\n\nlength (two times), offset, delimiter\n' % (utils.str2hex(item_str)) res = res + '%s decimal: %3d # length 1\n' % (utils.int2hex_str(item['len1']), item['len1']) res = res + '%s decimal: %3d # length 2\n' % (utils.int2hex_str(item['len2']), item['len2']) res = res + '%s decimal: %3d # offset\n' % (utils.int2hex_str(item['offset']), item['offset']) res = res + '%s # delimiter (two zeros)\n\n' % utils.str2hex(item_str[-2:]) item['string'] = res return item
def socks5_handler(self): sock = self.conn socks_version = ord(self.read_bytes(1)) auth_mode_num = ord(self.read_bytes(1)) data = self.read_bytes(auth_mode_num) sock.send(b"\x05\x00") # socks version 5, no auth needed. try: data = self.read_bytes(4) except Exception as e: xlog.debug("socks5 auth num:%d, list:%s", auth_mode_num, utils.str2hex(data)) xlog.warn("socks5 protocol error:%r", e) return socks_version = ord(data[0:1]) if socks_version != 5: xlog.warn("request version:%d error", socks_version) return command = ord(data[1:2]) if command != 1: # 1. Tcp connect xlog.warn("request not supported command mode:%d", command) sock.send(b"\x05\x07\x00\x01") # Command not supported return addrtype_pack = data[3:4] addrtype = ord(addrtype_pack) if addrtype == 1: # IPv4 addr_pack = self.read_bytes(4) addr = socket.inet_ntoa(addr_pack) elif addrtype == 3: # Domain name domain_len_pack = self.read_bytes(1)[0:1] domain_len = ord(domain_len_pack) domain = self.read_bytes(domain_len) addr_pack = domain_len_pack + domain addr = domain elif addrtype == 4: # IPv6 addr_pack = self.read_bytes(16) addr = socket.inet_ntop(socket.AF_INET6, addr_pack) else: xlog.warn("request address type unknown:%d", addrtype) sock.send(b"\x05\x07\x00\x01") # Command not supported return port = struct.unpack('>H', self.rfile.read(2))[0] # xlog.debug("socks5 %r connect to %s:%d", self.client_address, addr, port) reply = b"\x05\x00\x00" + addrtype_pack + addr_pack + struct.pack( ">H", port) sock.send(reply) if addrtype in [1, 4]: handle_ip_proxy(sock, addr, port, self.client_address) else: handle_domain_proxy(sock, addr, port, self.client_address)
def socks5_handler(self): sock = self.conn socks_version = ord(self.read_bytes(1)) auth_mode_num = ord(self.read_bytes(1)) data = self.read_bytes(auth_mode_num) sock.send(b"\x05\x00") # socks version 5, no auth needed. try: data = self.read_bytes(4) except Exception as e: xlog.debug("socks5 auth num:%d, list:%s", auth_mode_num, utils.str2hex(data)) xlog.warn("socks5 protocol error:%r", e) return socks_version = ord(data[0]) if socks_version != 5: xlog.warn("request version:%d error", socks_version) return command = ord(data[1]) if command != 1: # 1. Tcp connect xlog.warn("request not supported command mode:%d", command) sock.send(b"\x05\x07\x00\x01") # Command not supported return addrtype_pack = data[3] addrtype = ord(addrtype_pack) if addrtype == 1: # IPv4 addr_pack = self.read_bytes(4) addr = socket.inet_ntoa(addr_pack) elif addrtype == 3: # Domain name domain_len_pack = self.read_bytes(1)[0] domain_len = ord(domain_len_pack) domain = self.read_bytes(domain_len) addr_pack = domain_len_pack + domain addr = domain elif addrtype == 4: # IPv6 addr_pack = self.read_bytes(16) addr = socket.inet_ntop(socket.AF_INET6, addr_pack) else: xlog.warn("request address type unknown:%d", addrtype) sock.send(b"\x05\x07\x00\x01") # Command not supported return port = struct.unpack('>H', self.rfile.read(2))[0] # xlog.debug("socks5 %r connect to %s:%d", self.client_address, addr, port) reply = b"\x05\x00\x00" + addrtype_pack + addr_pack + struct.pack(">H", port) sock.send(reply) if addrtype in [1, 4]: handle_ip_proxy(sock, addr, port, self.client_address) else: handle_domain_proxy(sock, addr, port, self.client_address)
def item(item_str): "" item = {} res = '' item['len1'] = utils.bytes2int(item_str[0:2]) item['len2'] = utils.bytes2int(item_str[2:4]) item['offset'] = utils.bytes2int(item_str[4:6]) res = res + '%s\n\nlength (two times), offset, delimiter\n' % ( utils.str2hex(item_str)) res = res + '%s decimal: %3d # length 1\n' % (utils.int2hex_str( item['len1']), item['len1']) res = res + '%s decimal: %3d # length 2\n' % (utils.int2hex_str( item['len2']), item['len2']) res = res + '%s decimal: %3d # offset\n' % (utils.int2hex_str( item['offset']), item['offset']) res = res + '%s # delimiter (two zeros)\n\n' % utils.str2hex( item_str[-2:]) item['string'] = res return item
def login_session(self): if len(g.server_host) == 0 or g.server_port == 0: return False try: start_time = time.time() magic = "P" pack_type = 1 upload_data_head = struct.pack("<cBB8sIHII", magic, g.protocol_version, pack_type, str(self.session_id), g.config.block_max_size, g.config.send_delay, g.config.windows_size, g.config.windows_ack) upload_data_head += struct.pack("<H", len(g.config.login_account)) + str(g.config.login_account) upload_data_head += struct.pack("<H", len(g.config.login_password)) + str(g.config.login_password) upload_post_data = encrypt_data(upload_data_head) http_client = HTTP_client((g.server_host, g.server_port), g.proxy, g.config.use_https, g.config.conn_life, cert=g.cert) content, status, heads = http_client.request(method="POST", path="data", data=upload_post_data, timeout=g.config.roundtrip_timeout) time_cost = time.time() - start_time if status != 200: g.last_api_error = "session server login fail:%r" % status xlog.warn("login session fail, status:%r", status) return False if len(content) < 6: xlog.error("login data len:%d fail", len(content)) return False info = decrypt_data(content) magic, protocol_version, pack_type, res, message_len = struct.unpack("<cBBBH", info[:6]) message = info[6:] if magic != "P" or protocol_version != 1 or pack_type != 1: xlog.error("login_session time:%d head error:%s", 1000 * time_cost, utils.str2hex(info[:6])) return False if res != 0: g.last_api_error = "session server login fail, code:%d msg:%s" % (res, message) xlog.warn("login_session time:%d fail, res:%d msg:%s", 1000 * time_cost, res, message) return False g.last_api_error = "" xlog.info("login_session time:%d msg:%s", 1000 * time_cost, message) return True except Exception as e: xlog.exception("login_session e:%r", e) return False
def flags(flag_str): "" res = '' res = res + '%s\n\n' % utils.str2hex(flag_str) flags = utils.bytes2int(flag_str[0:2]) res = res + '%s # flags\n' % (utils.int2hex_str(flags)) res = res + 'Binary:\nlayout 87654321 87654321\n' res = res + ' %s %s\n' % (utils.byte2bin_str(flag_str[1]), utils.byte2bin_str(flag_str[0])) flags2 = utils.bytes2int(flag_str[2:4]) res = res + '%s # more flags ???\n' % (utils.int2hex_str(flags2)) res = res + 'Binary:\nlayout 87654321 87654321\n' res = res + ' %s %s\n' % (utils.byte2bin_str(flag_str[3]), utils.byte2bin_str(flag_str[2])) #res = res + '%s # delimiter ???\n' % m_hex[(cur + 2) * 2: (cur + 4) * 2] return res
def handle(self): try: # xlog.debug('Connected from %r', self.client_address) socks_version = self.read_bytes(1) if socks_version == "\x04": self.socks4_handler() elif socks_version == "\x05": self.socks5_handler() elif socks_version == "C": self.https_handler() else: xlog.warn("socks version:%s not supported", utils.str2hex(socks_version)) return except socket.error as e: xlog.exception("socks handler read error %r", e) except Exception as e: xlog.exception("any err:%r", e)
def flags(flag_str): "" res = '' res = res + '%s\n\n' % utils.str2hex(flag_str) flags = utils.bytes2int(flag_str[0:2]) res = res + '%s # flags\n' % (utils.int2hex_str(flags)) res = res + 'Binary:\nlayout 87654321 87654321\n' res = res + ' %s %s\n' % (utils.byte2bin_str( flag_str[1]), utils.byte2bin_str(flag_str[0])) flags2 = utils.bytes2int(flag_str[2:4]) res = res + '%s # more flags ???\n' % ( utils.int2hex_str(flags2)) res = res + 'Binary:\nlayout 87654321 87654321\n' res = res + ' %s %s\n' % (utils.byte2bin_str( flag_str[3]), utils.byte2bin_str(flag_str[2])) #res = res + '%s # delimiter ???\n' % m_hex[(cur + 2) * 2: (cur + 4) * 2] return res
def handle(self): self.__class__.handle_num += 1 try: dst = self.conn.getsockopt(socket.SOL_IP, SO_ORIGINAL_DST, 16) dst_port, srv_ip = struct.unpack("!2xH4s8x", dst) ip_str = socket.inet_ntoa(srv_ip) if dst_port != g.config.proxy_port and not utils.is_private_ip( ip_str): xlog.debug("Redirect to:%s:%d from:%s", ip_str, dst_port, self.client_address) return handle_ip_proxy(self.conn, ip_str, dst_port, self.client_address) except: pass sockets = [self.conn] try: r, w, e = select.select(sockets, [], []) socks_version = self.conn.recv(1, socket.MSG_PEEK) if not socks_version: return if socks_version == "\x04": self.socks4_handler() elif socks_version == "\x05": self.socks5_handler() elif socks_version == "C": self.https_handler() elif socks_version in ["G", "P", "D", "O", "H", "T"]: self.http_handler() else: xlog.warn("socks version:%s[%s] not supported", socks_version, utils.str2hex(socks_version)) return except socket.error as e: xlog.warn('socks handler read error:%r', e) except Exception as e: xlog.exception("any err:%r", e)
def debug_message1(msg): "" m_ = base64.decodestring(msg) m_hex = utils.str2hex(m_) res = '' res = res + '==============================================================\n' res = res + 'NTLM Message 1 report:\n' res = res + '---------------------------------\n' res = res + 'Base64: %s\n' % msg res = res + 'String: %s\n' % utils.str2prn_str(m_) res = res + 'Hex: %s\n' % m_hex cur = 0 res = res + '---------------------------------\n' cur_len = 12 res = res + 'Header %d/%d:\n%s\n\n' % (cur, cur_len, m_hex[0:24]) res = res + '%s\nmethod name 0/8\n%s # C string\n\n' % ( m_hex[0:16], utils.str2prn_str(m_[0:8])) res = res + '0x%s%s # message type\n' % (m_hex[18:20], m_hex[16:18]) res = res + '%s # delimiter (zeros)\n' % m_hex[20:24] cur = cur + cur_len res = res + '---------------------------------\n' cur_len = 4 res = res + 'Flags %d/%d\n' % (cur, cur_len) res = res + flags(m_[cur:cur + cur_len]) cur = cur + cur_len res = res + '---------------------------------\n' cur_len = len(m_) - cur res = res + 'Rest of the message %d/%d:\n' % (cur, cur_len) res = res + unknown_part(m_[cur:cur + cur_len]) res = res + '\nEnd of message 1 report.\n' return res
def debug_message1(msg): "" m_ = base64.decodestring(msg) m_hex = utils.str2hex(m_) res = '' res = res + '==============================================================\n' res = res + 'NTLM Message 1 report:\n' res = res + '---------------------------------\n' res = res + 'Base64: %s\n' % msg res = res + 'String: %s\n' % utils.str2prn_str(m_) res = res + 'Hex: %s\n' % m_hex cur = 0 res = res + '---------------------------------\n' cur_len = 12 res = res + 'Header %d/%d:\n%s\n\n' % (cur, cur_len, m_hex[0:24]) res = res + '%s\nmethod name 0/8\n%s # C string\n\n' % (m_hex[0:16], utils.str2prn_str(m_[0:8])) res = res + '0x%s%s # message type\n' % (m_hex[18:20], m_hex[16:18]) res = res + '%s # delimiter (zeros)\n' % m_hex[20:24] cur = cur + cur_len res = res + '---------------------------------\n' cur_len = 4 res = res + 'Flags %d/%d\n' % (cur, cur_len) res = res + flags(m_[cur: cur + cur_len]) cur = cur + cur_len res = res + '---------------------------------\n' cur_len = len(m_) - cur res = res + 'Rest of the message %d/%d:\n' % (cur, cur_len) res = res + unknown_part(m_[cur: cur + cur_len]) res = res + '\nEnd of message 1 report.\n' return res
def handle(self): self.__class__.handle_num += 1 try: dst = self.conn.getsockopt(socket.SOL_IP, SO_ORIGINAL_DST, 16) dst_port, srv_ip = struct.unpack("!2xH4s8x", dst) ip_str = socket.inet_ntoa(srv_ip) if dst_port != g.config.proxy_port and not utils.is_private_ip(ip_str): xlog.debug("Redirect to:%s:%d from:%s", ip_str, dst_port, self.client_address) return handle_ip_proxy(self.conn, ip_str, dst_port, self.client_address) except: pass sockets = [self.conn] try: r, w, e = select.select(sockets, [], []) socks_version = self.conn.recv(1, socket.MSG_PEEK) if not socks_version: return if socks_version == "\x04": self.socks4_handler() elif socks_version == "\x05": self.socks5_handler() elif socks_version == "C": self.https_handler() elif socks_version in ["G", "P", "D", "O", "H", "T"]: self.http_handler() else: xlog.warn("socks version:%s[%s] not supported", socks_version, utils.str2hex(socks_version)) return except socket.error as e: xlog.warn('socks handler read error:%r', e) except Exception as e: xlog.exception("any err:%r", e)
def debug_message2(msg): "" m_ = base64.decodestring(msg) m_hex = utils.str2hex(m_) res = '' res = res + '==============================================================\n' res = res + 'NTLM Message 2 report:\n' res = res + '---------------------------------\n' res = res + 'Base64: %s\n' % msg res = res + 'String: %s\n' % utils.str2prn_str(m_) res = res + 'Hex: %s\n' % m_hex cur = 0 res = res + '---------------------------------\n' cur_len = 12 res = res + 'Header %d/%d:\n%s\n\n' % (cur, cur_len, m_hex[0:24]) res = res + '%s\nmethod name 0/8\n%s # C string\n\n' % (m_hex[0:16], utils.str2prn_str(m_[0:8])) res = res + '0x%s%s # message type\n' % (m_hex[18:20], m_hex[16:18]) res = res + '%s # delimiter (zeros)\n' % m_hex[20:24] cur = cur + cur_len res = res + '---------------------------------\n' cur_len = 8 res = res + 'Lengths and Positions %d/%d\n%s\n\n' % (cur, cur_len, m_hex[cur * 2 :(cur + cur_len) * 2]) cur_len = 8 res = res + 'Domain ??? %d/%d\n' % (cur, cur_len) dom = item(m_[cur:cur+cur_len]) res = res + dom['string'] cur = cur + cur_len res = res + '---------------------------------\n' cur_len = 4 res = res + 'Flags %d/%d\n' % (cur, cur_len) res = res + flags(m_[cur: cur + cur_len]) cur = cur + cur_len res = res + '---------------------------------\n' cur_len = 8 res = res + 'NONCE %d/%d\n%s\n\n' % (cur, cur_len, m_hex[cur * 2 :(cur + cur_len) * 2]) cur = cur + cur_len res = res + '---------------------------------\n' cur_len = dom['offset'] - cur res = res + 'Unknown data %d/%d:\n' % (cur, cur_len) res = res + unknown_part(m_[cur: cur + cur_len]) cur = cur + cur_len res = res + '---------------------------------\n' cur_len = dom['len1'] res = res + 'Domain ??? %d/%d:\n' % (cur, cur_len) res = res + 'Hex: %s\n' % m_hex[cur * 2: (cur + cur_len) * 2] res = res + 'String: %s\n\n' % utils.str2prn_str(m_[cur : cur + cur_len]) cur = cur + cur_len res = res + '---------------------------------\n' cur_len = len(m_) - cur res = res + 'Rest of the message %d/%d:\n' % (cur, cur_len) res = res + unknown_part(m_[cur: cur + cur_len]) res = res + '\nEnd of message 2 report.\n' return res
def debug_message2(msg): "" m_ = base64.decodestring(msg) m_hex = utils.str2hex(m_) res = '' res = res + '==============================================================\n' res = res + 'NTLM Message 2 report:\n' res = res + '---------------------------------\n' res = res + 'Base64: %s\n' % msg res = res + 'String: %s\n' % utils.str2prn_str(m_) res = res + 'Hex: %s\n' % m_hex cur = 0 res = res + '---------------------------------\n' cur_len = 12 res = res + 'Header %d/%d:\n%s\n\n' % (cur, cur_len, m_hex[0:24]) res = res + '%s\nmethod name 0/8\n%s # C string\n\n' % ( m_hex[0:16], utils.str2prn_str(m_[0:8])) res = res + '0x%s%s # message type\n' % (m_hex[18:20], m_hex[16:18]) res = res + '%s # delimiter (zeros)\n' % m_hex[20:24] cur = cur + cur_len res = res + '---------------------------------\n' cur_len = 8 res = res + 'Lengths and Positions %d/%d\n%s\n\n' % ( cur, cur_len, m_hex[cur * 2:(cur + cur_len) * 2]) cur_len = 8 res = res + 'Domain ??? %d/%d\n' % (cur, cur_len) dom = item(m_[cur:cur + cur_len]) res = res + dom['string'] cur = cur + cur_len res = res + '---------------------------------\n' cur_len = 4 res = res + 'Flags %d/%d\n' % (cur, cur_len) res = res + flags(m_[cur:cur + cur_len]) cur = cur + cur_len res = res + '---------------------------------\n' cur_len = 8 res = res + 'NONCE %d/%d\n%s\n\n' % (cur, cur_len, m_hex[cur * 2:(cur + cur_len) * 2]) cur = cur + cur_len res = res + '---------------------------------\n' cur_len = dom['offset'] - cur res = res + 'Unknown data %d/%d:\n' % (cur, cur_len) res = res + unknown_part(m_[cur:cur + cur_len]) cur = cur + cur_len res = res + '---------------------------------\n' cur_len = dom['len1'] res = res + 'Domain ??? %d/%d:\n' % (cur, cur_len) res = res + 'Hex: %s\n' % m_hex[cur * 2:(cur + cur_len) * 2] res = res + 'String: %s\n\n' % utils.str2prn_str(m_[cur:cur + cur_len]) cur = cur + cur_len res = res + '---------------------------------\n' cur_len = len(m_) - cur res = res + 'Rest of the message %d/%d:\n' % (cur, cur_len) res = res + unknown_part(m_[cur:cur + cur_len]) res = res + '\nEnd of message 2 report.\n' return res
def login_session(self): if len(g.server_host) == 0: return False start_time = time.time() while time.time() - start_time < 30: try: start_time = time.time() magic = "P" pack_type = 1 upload_data_head = struct.pack( "<cBB8sIHIIHH", magic, g.protocol_version, pack_type, str(self.session_id), g.config.max_payload, g.config.send_delay, g.config.windows_size, g.config.windows_ack, g.config.resend_timeout, g.config.ack_delay) upload_data_head += struct.pack( "<H", len(g.config.login_account)) + str( g.config.login_account) upload_data_head += struct.pack( "<H", len(g.config.login_password)) + str( g.config.login_password) upload_post_data = encrypt_data(upload_data_head) content, status, response = g.http_client.request( method="POST", host=g.server_host, path="/data", data=upload_post_data, timeout=g.config.network_timeout) time_cost = time.time() - start_time if status == 521: g.last_api_error = "session server is down." xlog.warn( "login session server is down, try get new server.") g.server_host = None return False if status != 200: g.last_api_error = "session server login fail:%r" % status xlog.warn("login session fail, status:%r", status) continue if len(content) < 6: g.last_api_error = "session server protocol fail, login res len:%d" % len( content) xlog.error("login data len:%d fail", len(content)) continue info = decrypt_data(content) magic, protocol_version, pack_type, res, message_len = struct.unpack( "<cBBBH", info[:6]) message = info[6:] if magic != "P" or protocol_version != g.protocol_version or pack_type != 1: xlog.error("login_session time:%d head error:%s", 1000 * time_cost, utils.str2hex(info[:6])) return False if res != 0: g.last_api_error = "session server login fail, code:%d msg:%s" % ( res, message) xlog.warn("login_session time:%d fail, res:%d msg:%s", 1000 * time_cost, res, message) return False g.last_api_error = "" xlog.info("login_session %s time:%d msg:%s", self.session_id, 1000 * time_cost, message) return True except Exception as e: xlog.exception("login_session e:%r", e) time.sleep(1) return False
def normal_roundtrip_worker(self): while self.running: data, ack = self.get_send_data() if not self.running: return send_data_len = len(data) send_ack_len = len(ack) transfer_no = self.get_transfer_no() magic = "P" pack_type = 2 if self.on_road_num > g.config.concurent_thread_num * 0.6: server_timeout = 0 else: server_timeout = g.config.roundtrip_timeout request_session_id = self.session_id upload_data_head = struct.pack("<cBB8sIBIH", magic, g.protocol_version, pack_type, str(self.session_id), transfer_no, server_timeout, send_data_len, send_ack_len) upload_post_buf = base_container.WriteBuffer(upload_data_head) upload_post_buf.append(data) upload_post_buf.append(ack) upload_post_data = str(upload_post_buf) upload_post_data = encrypt_data(upload_post_data) self.last_send_time = time.time() sleep_time = 1 start_time = time.time() with self.lock: self.on_road_num += 1 self.transfer_list[transfer_no] = {} self.transfer_list[transfer_no]["stat"] = "request" self.transfer_list[transfer_no]["start"] = start_time #xlog.debug("start roundtrip transfer_no:%d send_data_len:%d ack_len:%d timeout:%d", # transfer_no, send_data_len, send_ack_len, server_timeout) try: content, status, response = g.http_client.request( method="POST", host=g.server_host, path="/data?tid=%d" % transfer_no, data=upload_post_data, headers={"Content-Length": str(len(upload_post_data))}, timeout=server_timeout + g.config.network_timeout) traffic = len(upload_post_data) + len(content) + 645 self.traffic += traffic g.quota -= traffic except Exception as e: xlog.exception("request except:%r ", e) time.sleep(sleep_time) continue finally: with self.lock: self.on_road_num -= 1 try: if transfer_no in self.transfer_list: del self.transfer_list[transfer_no] except: pass g.stat["roundtrip_num"] += 1 roundtrip_time = (time.time() - start_time) * 1000 if status == 521: xlog.warn("X-tunnel server is down, try get new server.") g.server_host = None self.stop() login_process() return if status != 200: xlog.warn( "roundtrip time:%d transfer_no:%d send:%d status:%r ", roundtrip_time, transfer_no, send_data_len, status) time.sleep(sleep_time) continue recv_len = len(content) if recv_len < 6: xlog.warn( "roundtrip time:%d transfer_no:%d send:%d recv:%d Head", roundtrip_time, transfer_no, send_data_len, recv_len) continue content = decrypt_data(content) payload = base_container.ReadBuffer(content) magic, version, pack_type = struct.unpack("<cBB", payload.get(3)) if magic != "P" or version != g.protocol_version: xlog.warn("get data head:%s", utils.str2hex(content[:2])) time.sleep(sleep_time) continue if pack_type == 3: # error report error_code, message_len = struct.unpack("<BH", payload.get(3)) message = payload.get(message_len) # xlog.warn("report code:%d, msg:%s", error_code, message) if error_code == 1: # no quota xlog.warn("x_server error:no quota") self.stop() return elif error_code == 2: # unpack error xlog.warn( "roundtrip time:%d transfer_no:%d send:%d recv:%d unpack_error:%s", roundtrip_time, transfer_no, send_data_len, len(content), message) continue elif error_code == 3: # session not exist if self.session_id == request_session_id: xlog.warn( "server session_id:%s not exist, reset session.", request_session_id) self.reset() return else: continue else: xlog.error("unknown error code:%d, message:%s", error_code, message) time.sleep(sleep_time) continue if pack_type != 2: # normal download traffic pack xlog.error("pack type:%d", pack_type) time.sleep(100) continue time_cost, server_send_pool_size, data_len, ack_len = struct.unpack( "<IIIH", payload.get(14)) xlog.debug( "roundtrip time:%d cost:%d transfer_no:%d send:%d rcv:%d ", roundtrip_time, time_cost, transfer_no, send_data_len, len(content)) self.trigger_more(server_send_pool_size) rtt = roundtrip_time - time_cost rtt = max(100, rtt) speed = (send_data_len + len(content) + 400) / rtt response.worker.update_debug_data(rtt, send_data_len, len(content), speed) if rtt > 8000: xlog.debug("rtt:%d speed:%d trace:%s", rtt, speed, response.worker.get_trace()) xlog.debug("task trace:%s", response.task.get_trace()) g.stat["slow_roundtrip"] += 1 try: data = payload.get_buf(data_len) ack = payload.get_buf(ack_len) except: xlog.debug("data not enough") continue try: self.roundtrip_process(data, ack) self.last_receive_time = time.time() except Exception as e: xlog.exception("data process:%r", e) continue xlog.info("roundtrip thread exit")
def debug_message3(msg): "" m_ = base64.decodestring(msg) m_hex = utils.str2hex(m_) res = '' res = res + '==============================================================\n' res = res + 'NTLM Message 3 report:\n' res = res + '---------------------------------\n' res = res + 'Base64: %s\n' % msg res = res + 'String: %s\n' % utils.str2prn_str(m_) res = res + 'Hex: %s\n' % m_hex cur = 0 res = res + '---------------------------------\n' cur_len = 12 res = res + 'Header %d/%d:\n%s\n\n' % (cur, cur_len, m_hex[0:24]) res = res + '%s\nmethod name 0/8\n%s # C string\n\n' % ( m_hex[0:16], utils.str2prn_str(m_[0:8])) res = res + '0x%s%s # message type\n' % (m_hex[18:20], m_hex[16:18]) res = res + '%s # delimiter (zeros)\n' % m_hex[20:24] cur = cur + cur_len res = res + '---------------------------------\n' cur_len = 48 res = res + 'Lengths and Positions %d/%d\n%s\n\n' % ( cur, cur_len, m_hex[cur * 2:(cur + cur_len) * 2]) cur_len = 8 res = res + 'LAN Manager response %d/%d\n' % (cur, cur_len) lmr = item(m_[cur:cur + cur_len]) res = res + lmr['string'] cur = cur + cur_len cur_len = 8 res = res + 'NT response %d/%d\n' % (cur, cur_len) ntr = item(m_[cur:cur + cur_len]) res = res + ntr['string'] cur = cur + cur_len cur_len = 8 res = res + 'Domain string %d/%d\n' % (cur, cur_len) dom = item(m_[cur:cur + cur_len]) res = res + dom['string'] cur = cur + cur_len cur_len = 8 res = res + 'User string %d/%d\n' % (cur, cur_len) user = item(m_[cur:cur + cur_len]) res = res + user['string'] cur = cur + cur_len cur_len = 8 res = res + 'Host string %d/%d\n' % (cur, cur_len) host = item(m_[cur:cur + cur_len]) res = res + host['string'] cur = cur + cur_len cur_len = 8 res = res + 'Unknow item record %d/%d\n' % (cur, cur_len) unknown = item(m_[cur:cur + cur_len]) res = res + unknown['string'] cur = cur + cur_len res = res + '---------------------------------\n' cur_len = 4 res = res + 'Flags %d/%d\n' % (cur, cur_len) res = res + flags(m_[cur:cur + cur_len]) cur = cur + cur_len res = res + '---------------------------------\n' cur_len = dom['len1'] + user['len1'] + host['len1'] res = res + 'Domain, User, Host strings %d/%d\n%s\n%s\n\n' % ( cur, cur_len, m_hex[cur * 2:(cur + cur_len) * 2], utils.str2prn_str(m_[cur:cur + cur_len])) cur_len = dom['len1'] res = res + '%s\n' % m_hex[cur * 2:(cur + cur_len) * 2] res = res + 'Domain name %d/%d:\n' % (cur, cur_len) res = res + '%s\n\n' % (utils.str2prn_str(m_[cur:(cur + cur_len)])) cur = cur + cur_len cur_len = user['len1'] res = res + '%s\n' % m_hex[cur * 2:(cur + cur_len) * 2] res = res + 'User name %d/%d:\n' % (cur, cur_len) res = res + '%s\n\n' % (utils.str2prn_str(m_[cur:(cur + cur_len)])) cur = cur + cur_len cur_len = host['len1'] res = res + '%s\n' % m_hex[cur * 2:(cur + cur_len) * 2] res = res + 'Host name %d/%d:\n' % (cur, cur_len) res = res + '%s\n\n' % (utils.str2prn_str(m_[cur:(cur + cur_len)])) cur = cur + cur_len res = res + '---------------------------------\n' cur_len = lmr['len1'] res = res + 'LAN Manager response %d/%d\n%s\n\n' % ( cur, cur_len, m_hex[cur * 2:(cur + cur_len) * 2]) cur = cur + cur_len res = res + '---------------------------------\n' cur_len = ntr['len1'] res = res + 'NT response %d/%d\n%s\n\n' % ( cur, cur_len, m_hex[cur * 2:(cur + cur_len) * 2]) cur = cur + cur_len res = res + '---------------------------------\n' cur_len = len(m_) - cur res = res + 'Rest of the message %d/%d:\n' % (cur, cur_len) res = res + unknown_part(m_[cur:cur + cur_len]) res = res + '\nEnd of message 3 report.\n' return res
def normal_roundtrip_worker(self): while self.running: data, ack = self.get_send_data() if not self.running: return send_data_len = len(data) send_ack_len = len(ack) transfer_no = self.get_transfer_no() magic = "P" pack_type = 2 if self.on_road_num > g.config.concurent_thread_num * 0.6: server_timeout = 0 else: server_timeout = g.config.roundtrip_timeout request_session_id = self.session_id upload_data_head = struct.pack("<cBB8sIBIH", magic, g.protocol_version, pack_type, str(self.session_id), transfer_no, server_timeout, send_data_len, send_ack_len) upload_post_buf = base_container.WriteBuffer(upload_data_head) upload_post_buf.append(data) upload_post_buf.append(ack) upload_post_data = str(upload_post_buf) upload_post_data = encrypt_data(upload_post_data) self.last_send_time = time.time() sleep_time = 1 start_time = time.time() with self.lock: self.on_road_num += 1 self.transfer_list[transfer_no] = {} self.transfer_list[transfer_no]["stat"] = "request" self.transfer_list[transfer_no]["start"] = start_time #xlog.debug("start roundtrip transfer_no:%d send_data_len:%d ack_len:%d timeout:%d", # transfer_no, send_data_len, send_ack_len, server_timeout) try: content, status, response = g.http_client.request(method="POST", host=g.server_host, path="/data", data=upload_post_data, headers={"Content-Length": str(len(upload_post_data))}, timeout=server_timeout + g.config.network_timeout) traffic = len(upload_post_data) + len(content) + 645 self.traffic += traffic g.quota -= traffic except Exception as e: xlog.exception("request except:%r ", e) time.sleep(sleep_time) continue finally: with self.lock: self.on_road_num -= 1 try: if transfer_no in self.transfer_list: del self.transfer_list[transfer_no] except: pass g.stat["roundtrip_num"] += 1 roundtrip_time = (time.time() - start_time) * 1000 if status == 521: xlog.warn("X-tunnel server is down, try get new server.") request_balance(update_server=True) self.reset() return if status != 200: xlog.warn("roundtrip time:%d transfer_no:%d send:%d status:%r ", roundtrip_time, transfer_no, send_data_len, status) time.sleep(sleep_time) continue recv_len = len(content) if recv_len < 6: xlog.warn("roundtrip time:%d transfer_no:%d send:%d recv:%d Head", roundtrip_time, transfer_no, send_data_len, recv_len) continue content = decrypt_data(content) payload = base_container.ReadBuffer(content) magic, version, pack_type = struct.unpack("<cBB", payload.get(3)) if magic != "P" or version != g.protocol_version: xlog.warn("get data head:%s", utils.str2hex(content[:2])) time.sleep(sleep_time) continue if pack_type == 3: # error report error_code, message_len = struct.unpack("<BH", payload.get(3)) message = payload.get(message_len) # xlog.warn("report code:%d, msg:%s", error_code, message) if error_code == 1: # no quota xlog.warn("x_server error:no quota") self.stop() return elif error_code == 2: # unpack error xlog.warn("roundtrip time:%d transfer_no:%d send:%d recv:%d unpack_error:%s", roundtrip_time, transfer_no, send_data_len, len(content), message) continue elif error_code == 3: # session not exist if self.session_id == request_session_id: xlog.warn("server session_id:%s not exist, reset session.", request_session_id) self.reset() return else: continue else: xlog.error("unknown error code:%d, message:%s", error_code, message) time.sleep(sleep_time) continue if pack_type != 2: # normal download traffic pack xlog.error("pack type:%d", pack_type) time.sleep(100) continue time_cost, server_send_pool_size, data_len, ack_len = struct.unpack("<IIIH", payload.get(14)) xlog.debug("roundtrip time:%d cost:%d transfer_no:%d send:%d rcv:%d ", roundtrip_time, time_cost, transfer_no, send_data_len, len(content)) self.trigger_more(server_send_pool_size) rtt = roundtrip_time - time_cost rtt = max(100, rtt) speed = (send_data_len + len(content) + 400) / rtt response.worker.update_debug_data(rtt, send_data_len, len(content), speed) if rtt > 8000: xlog.debug("rtt:%d speed:%d trace:%s", rtt, speed, response.worker.get_trace()) xlog.debug("task trace:%s", response.task.get_trace()) g.stat["slow_roundtrip"] += 1 try: data = payload.get_buf(data_len) ack = payload.get_buf(ack_len) except: xlog.debug("data not enough") continue try: self.roundtrip_process(data, ack) self.last_receive_time = time.time() except Exception as e: xlog.exception("data process:%r", e) continue xlog.info("roundtrip thread exit")
def login_session(self): if len(g.server_host) == 0: return False start_time = time.time() while time.time() - start_time < 30: try: start_time = time.time() magic = "P" pack_type = 1 upload_data_head = struct.pack("<cBB8sIHIIHH", magic, g.protocol_version, pack_type, str(self.session_id), g.config.max_payload, g.config.send_delay, g.config.windows_size, g.config.windows_ack, g.config.resend_timeout, g.config.ack_delay) upload_data_head += struct.pack("<H", len(g.config.login_account)) + str(g.config.login_account) upload_data_head += struct.pack("<H", len(g.config.login_password)) + str(g.config.login_password) upload_post_data = encrypt_data(upload_data_head) content, status, response = g.http_client.request(method="POST", host=g.server_host, path="/data", data=upload_post_data, timeout=g.config.network_timeout) time_cost = time.time() - start_time if status == 521: g.last_api_error = "session server is down." xlog.warn("login session server is down, try get new server.") request_balance(update_server=True) continue if status != 200: g.last_api_error = "session server login fail:%r" % status xlog.warn("login session fail, status:%r", status) continue if len(content) < 6: g.last_api_error = "session server protocol fail, login res len:%d" % len(content) xlog.error("login data len:%d fail", len(content)) continue info = decrypt_data(content) magic, protocol_version, pack_type, res, message_len = struct.unpack("<cBBBH", info[:6]) message = info[6:] if magic != "P" or protocol_version != g.protocol_version or pack_type != 1: xlog.error("login_session time:%d head error:%s", 1000 * time_cost, utils.str2hex(info[:6])) return False if res != 0: g.last_api_error = "session server login fail, code:%d msg:%s" % (res, message) xlog.warn("login_session time:%d fail, res:%d msg:%s", 1000 * time_cost, res, message) return False g.last_api_error = "" xlog.info("login_session %s time:%d msg:%s", self.session_id, 1000 * time_cost, message) return True except Exception as e: xlog.exception("login_session e:%r", e) time.sleep(1) return False
def debug_message3(msg): "" m_ = base64.decodestring(msg) m_hex = utils.str2hex(m_) res = '' res = res + '==============================================================\n' res = res + 'NTLM Message 3 report:\n' res = res + '---------------------------------\n' res = res + 'Base64: %s\n' % msg res = res + 'String: %s\n' % utils.str2prn_str(m_) res = res + 'Hex: %s\n' % m_hex cur = 0 res = res + '---------------------------------\n' cur_len = 12 res = res + 'Header %d/%d:\n%s\n\n' % (cur, cur_len, m_hex[0:24]) res = res + '%s\nmethod name 0/8\n%s # C string\n\n' % (m_hex[0:16], utils.str2prn_str(m_[0:8])) res = res + '0x%s%s # message type\n' % (m_hex[18:20], m_hex[16:18]) res = res + '%s # delimiter (zeros)\n' % m_hex[20:24] cur = cur + cur_len res = res + '---------------------------------\n' cur_len = 48 res = res + 'Lengths and Positions %d/%d\n%s\n\n' % (cur, cur_len, m_hex[cur * 2 :(cur + cur_len) * 2]) cur_len = 8 res = res + 'LAN Manager response %d/%d\n' % (cur, cur_len) lmr = item(m_[cur:cur+cur_len]) res = res + lmr['string'] cur = cur + cur_len cur_len = 8 res = res + 'NT response %d/%d\n' % (cur, cur_len) ntr = item(m_[cur:cur+cur_len]) res = res + ntr['string'] cur = cur + cur_len cur_len = 8 res = res + 'Domain string %d/%d\n' % (cur, cur_len) dom = item(m_[cur:cur+cur_len]) res = res + dom['string'] cur = cur + cur_len cur_len = 8 res = res + 'User string %d/%d\n' % (cur, cur_len) user = item(m_[cur:cur+cur_len]) res = res + user['string'] cur = cur + cur_len cur_len = 8 res = res + 'Host string %d/%d\n' % (cur, cur_len) host = item(m_[cur:cur+cur_len]) res = res + host['string'] cur = cur + cur_len cur_len = 8 res = res + 'Unknow item record %d/%d\n' % (cur, cur_len) unknown = item(m_[cur:cur+cur_len]) res = res + unknown['string'] cur = cur + cur_len res = res + '---------------------------------\n' cur_len = 4 res = res + 'Flags %d/%d\n' % (cur, cur_len) res = res + flags(m_[cur: cur + cur_len]) cur = cur + cur_len res = res + '---------------------------------\n' cur_len = dom['len1'] + user['len1'] + host['len1'] res = res + 'Domain, User, Host strings %d/%d\n%s\n%s\n\n' % (cur, cur_len, m_hex[cur * 2 :(cur + cur_len) * 2], utils.str2prn_str(m_[cur:cur + cur_len])) cur_len = dom['len1'] res = res + '%s\n' % m_hex[cur * 2: (cur + cur_len) * 2] res = res + 'Domain name %d/%d:\n' % (cur, cur_len) res = res + '%s\n\n' % (utils.str2prn_str(m_[cur: (cur + cur_len)])) cur = cur + cur_len cur_len = user['len1'] res = res + '%s\n' % m_hex[cur * 2: (cur + cur_len) * 2] res = res + 'User name %d/%d:\n' % (cur, cur_len) res = res + '%s\n\n' % (utils.str2prn_str(m_[cur: (cur + cur_len)])) cur = cur + cur_len cur_len = host['len1'] res = res + '%s\n' % m_hex[cur * 2: (cur + cur_len) * 2] res = res + 'Host name %d/%d:\n' % (cur, cur_len) res = res + '%s\n\n' % (utils.str2prn_str(m_[cur: (cur + cur_len)])) cur = cur + cur_len res = res + '---------------------------------\n' cur_len = lmr['len1'] res = res + 'LAN Manager response %d/%d\n%s\n\n' % (cur, cur_len, m_hex[cur * 2 :(cur + cur_len) * 2]) cur = cur + cur_len res = res + '---------------------------------\n' cur_len = ntr['len1'] res = res + 'NT response %d/%d\n%s\n\n' % (cur, cur_len, m_hex[cur * 2 :(cur + cur_len) * 2]) cur = cur + cur_len res = res + '---------------------------------\n' cur_len = len(m_) - cur res = res + 'Rest of the message %d/%d:\n' % (cur, cur_len) res = res + unknown_part(m_[cur: cur + cur_len]) res = res + '\nEnd of message 3 report.\n' return res
def normal_roundtrip_worker(self, server_address): last_roundtrip_download_size = 0 while self.running: if self.on_road_num > g.config.concurent_thread_num * 0.9: block = True xlog.debug("block for too many on road") elif last_roundtrip_download_size > 0: block = False elif len(self.conn_list) > 0 and self.on_road_num < 1: # keep at least one pulling thread block = False elif len(self.conn_list) > 0 and time.time() - self.last_download_data_time < 120 and \ self.on_road_num < g.config.concurent_thread_num * 0.1: # busy, have data download block = False else: block = True # xlog.debug("block for none") if block: get_timeout = 24 * 3600 else: get_timeout = 0 # self.transfer_list[transfer_no]["stat"] = "get local data" upload_data, send_sn = self.upload_task_queue.get(get_timeout) transfer_no = self.get_transfer_no() self.transfer_list[transfer_no] = {} self.transfer_list[transfer_no]["sn"] = send_sn send_data_len = len(upload_data) upload_ack_data = self.ack_pool.get() send_ack_len = len(upload_ack_data) magic = "P" pack_type = 2 if self.on_road_num > g.config.concurent_thread_num * 0.8: server_timeout = 0 else: if last_roundtrip_download_size >= g.config.block_max_size: server_timeout = 1 else: server_timeout = g.config.roundtrip_timeout / 2 request_session_id = self.session_id upload_data_head = struct.pack("<cBB8sIIBIH", magic, g.protocol_version, pack_type, str(self.session_id), transfer_no, send_sn, server_timeout, send_data_len, send_ack_len) upload_post_buf = base_container.WriteBuffer(upload_data_head) upload_post_buf.append(upload_data) upload_post_buf.append(upload_ack_data) upload_post_data = str(upload_post_buf) upload_post_data = encrypt_data(upload_post_data) try_no = 0 while self.running: try_no += 1 sleep_time = min(try_no, 30) self.last_roundtrip_time = time.time() start_time = time.time() with self.mutex: self.on_road_num += 1 # xlog.debug("start roundtrip transfer_no:%d send_data_len:%d ack_len:%d", transfer_no, send_data_len, send_ack_len) try: self.transfer_list[transfer_no]["try"] = try_no self.transfer_list[transfer_no]["stat"] = "request" self.transfer_list[transfer_no]["start"] = time.time() content, status, response = g.http_client.request(method="POST", host=g.server_host, path="/data", data=upload_post_data, timeout=server_timeout + g.config.network_timeout) traffic = len(upload_post_data) + len(content) + 645 self.traffic += traffic g.quota -= traffic except Exception as e: xlog.exception("request except:%r retry %d", e, try_no) time.sleep(sleep_time) if transfer_no not in self.transfer_list: break else: continue finally: with self.mutex: self.on_road_num -= 1 g.stat["roundtrip_num"] += 1 if status == 521: xlog.warn("X-tunnel server is down, try get new server.") request_balance(update_server=True) self.reset() return if status == 405: # session_id not exist on server if self.running: if self.session_id == request_session_id: xlog.warn("server session_id:%s not exist, reset session.", request_session_id) self.reset() return elif status == 200: recv_len = len(content) if recv_len < 6: xlog.warn("roundtrip time:%d transfer_no:%d sn:%d send:%d len:%d status:%r retry:%d", (time.time() - start_time) * 1000, transfer_no, send_sn, send_data_len, len(content), status, try_no) continue content = decrypt_data(content) data = base_container.ReadBuffer(content) magic, version, pack_type = struct.unpack("<cBB", data.get(3)) if magic != "P" or version != g.protocol_version: xlog.error("get data head:%s", utils.str2hex(content[:2])) time.sleep(100) break if pack_type == 3: # error report error_code, message_len = struct.unpack("<BH", data.get(3)) message = data.get(message_len) xlog.warn("error report code:%d, msg:%s", error_code, message) if error_code == 1: # no quota xlog.warn("login x_server error:no quota") self.stop() return else: xlog.error("unknown error code:%d", error_code) return if pack_type != 2: # normal download traffic pack xlog.error("pack type:%d", pack_type) time.sleep(100) break sn, time_cost = struct.unpack("<II", data.get(8)) roundtrip_time = (time.time() - start_time) * 1000 xlog.debug( "roundtrip time:%d cost:%d transfer_no:%d send_sn:%d send:%d recv_sn:%d rcv:%d status:%r", roundtrip_time, time_cost, transfer_no, send_sn, send_data_len, sn, len(content), status) rtt = roundtrip_time - time_cost speed = (send_data_len + len(content) + 400) / rtt response.worker.update_debug_data(rtt, send_data_len, len(content), speed) if rtt > 8000: xlog.warn("rtt:%d speed:%d trace:%s", rtt, speed, response.worker.get_trace()) xlog.warn("task trace:%s", response.task.get_trace()) g.stat["slow_roundtrip"] += 1 data_len = len(data) if (sn > 0 and data_len == 0) or (sn == 0 and data_len > 0): xlog.warn("get sn:%d len:%d %s", sn, data_len, data) if send_data_len == 0 and \ data_len >= g.config.block_max_size and \ last_roundtrip_download_size >= g.config.block_max_size: if self.on_road_num < int(g.config.concurent_thread_num * 0.8): self.touch_roundtrip() if sn: self.last_download_data_time = time.time() last_roundtrip_download_size = data_len # xlog.debug("get sn:%d len:%d", sn, data_len) # self.download_order_queue.put(sn, data) try: self.download_data_processor(data) except Exception as e: xlog.warn("data process:%r", e) continue ack_pak = struct.pack("<Q", transfer_no) self.ack_pool.put(ack_pak) else: last_roundtrip_download_size = 0 break else: g.stat["timeout_roundtrip"] += 1 xlog.warn("roundtrip time:%d transfer_no:%d send_sn:%d send:%d status:%r retry:%d", (time.time() - start_time) * 1000, transfer_no, send_sn, send_data_len, status, try_no) time.sleep(sleep_time) try: if transfer_no in self.transfer_list: del self.transfer_list[transfer_no] except: pass xlog.info("roundtrip port:%d thread exit", server_address[1])
def normal_roundtrip_worker(self, server_address): last_roundtrip_download_size = 0 while self.running: if self.on_road_num > g.config.concurent_thread_num * 0.8: block = True elif last_roundtrip_download_size > g.config.block_max_size: block = False elif len(self.conn_list) > 0 and self.on_road_num < 1: # keep at least one pulling thread block = False elif len(self.conn_list) > 0 and time.time() - self.last_download_data_time < 120 and \ self.on_road_num < g.config.concurent_thread_num * 0.1: # busy, have data download block = False else: block = True if block: get_timeout = 24 * 3600 else: get_timeout = 0 # self.transfer_list[transfer_no]["stat"] = "get local data" upload_data, send_sn = self.upload_task_queue.get(get_timeout) transfer_no = self.get_transfer_no() self.transfer_list[transfer_no] = {} self.transfer_list[transfer_no]["sn"] = send_sn send_data_len = len(upload_data) upload_ack_data = self.ack_pool.get() send_ack_len = len(upload_ack_data) magic = "P" pack_type = 2 if self.on_road_num > g.config.concurent_thread_num * 0.8: server_timeout = 0 else: server_timeout = g.config.roundtrip_timeout / 2 request_session_id = self.session_id upload_data_head = struct.pack("<cBB8sIIBIH", magic, g.protocol_version, pack_type, str(self.session_id), transfer_no, send_sn, server_timeout, send_data_len, send_ack_len) upload_post_buf = base_container.WriteBuffer(upload_data_head) upload_post_buf.append(upload_data) upload_post_buf.append(upload_ack_data) upload_post_data = str(upload_post_buf) upload_post_data = encrypt_data(upload_post_data) try_no = 0 while self.running: try_no += 1 sleep_time = min(try_no, 30) self.last_roundtrip_time = time.time() start_time = time.time() with self.mutex: self.on_road_num += 1 # xlog.debug("start roundtrip transfer_no:%d send_data_len:%d ack_len:%d", transfer_no, send_data_len, send_ack_len) try: self.transfer_list[transfer_no]["try"] = try_no self.transfer_list[transfer_no]["stat"] = "request" self.transfer_list[transfer_no]["start"] = time.time() content, status, response = g.http_client.request(method="POST", host=g.server_host, path="/data", data=upload_post_data, timeout=g.config.roundtrip_timeout) traffic = len(upload_post_data) + len(content) + 645 self.traffic += traffic g.quota -= traffic except Exception as e: xlog.exception("request except:%r retry %d", e, try_no) time.sleep(sleep_time) if transfer_no not in self.transfer_list: break else: continue finally: with self.mutex: self.on_road_num -= 1 if status == 405: # session_id not exist on server if self.running: if self.session_id == request_session_id: xlog.warn("server session_id:%s not exist, reset session.", request_session_id) self.reset() return elif status == 200: recv_len = len(content) if recv_len < 6: xlog.error("roundtrip time:%d transfer_no:%d sn:%d send:%d len:%d status:%r retry:%d", (time.time() - start_time) * 1000, transfer_no, send_sn, send_data_len, len(content), status, try_no) continue content = decrypt_data(content) data = base_container.ReadBuffer(content) magic, version, pack_type = struct.unpack("<cBB", data.get(3)) if magic != "P" or version != g.protocol_version: xlog.error("get data head:%s", utils.str2hex(content[:2])) time.sleep(100) break if pack_type == 3: # error report error_code, message_len = struct.unpack("<BH", data.get(3)) message = data.get(message_len) xlog.warn("error report code:%d, msg:%s", error_code, message) if error_code == 1: # no quota xlog.warn("login x_server error:no quota") self.stop() return else: xlog.error("unknown error code:%d", error_code) return if pack_type != 2: # normal download traffic pack xlog.error("pack type:%d", pack_type) time.sleep(100) break sn, time_cost = struct.unpack("<II", data.get(8)) xlog.debug( "roundtrip time:%d cost:%d transfer_no:%d send_sn:%d send:%d recv_sn:%d rcv:%d status:%r", (time.time() - start_time) * 1000, time_cost, transfer_no, send_sn, send_data_len, sn, len(content), status) data_len = len(data) if (sn > 0 and data_len == 0) or (sn == 0 and data_len > 0): xlog.warn("get sn:%d len:%d %s", sn, data_len, data) if sn: self.last_download_data_time = time.time() last_roundtrip_download_size = data_len # xlog.debug("get sn:%d len:%d", sn, data_len) self.download_order_queue.put(sn, data) ack_pak = struct.pack("<Q", transfer_no) self.ack_pool.put(ack_pak) else: last_roundtrip_download_size = 0 if send_data_len == 0 and data_len > g.config.block_max_size: need_more_thread_num = int(g.config.concurent_thread_num * 0.5 - self.on_road_num) if need_more_thread_num > 0: for j in range(0, need_more_thread_num): if self.on_road_num > g.config.concurent_thread_num * 0.5: break self.touch_roundtrip() break else: xlog.warn("roundtrip time:%d transfer_no:%d send_sn:%d send:%d status:%r retry:%d", (time.time() - start_time) * 1000, transfer_no, send_sn, send_data_len, status, try_no) time.sleep(sleep_time) try: if transfer_no in self.transfer_list: del self.transfer_list[transfer_no] except: pass xlog.info("roundtrip port:%d thread exit", server_address[1])
def normal_roundtrip_worker(self, server_address): last_roundtrip_download_size = 0 http_client = HTTP_client(server_address, g.proxy, g.config.use_https, g.config.conn_life, cert=g.cert) while self.running: if self.on_road_num > g.config.concurent_thread_num * 0.8: block = True elif last_roundtrip_download_size > g.config.block_max_size: block = False elif len(self.conn_list) > 0 and self.on_road_num < 1: # keep at least one pulling thread block = False elif len(self.conn_list) > 0 and time.time() - self.last_download_data_time < 120 and \ self.on_road_num < g.config.concurent_thread_num * 0.1: # busy, have data download block = False else: block = True if block: get_timeout = 24 * 3600 else: get_timeout = 0 # self.transfer_list[transfer_no]["stat"] = "get local data" upload_data, send_sn = self.upload_task_queue.get(get_timeout) transfer_no = self.get_transfer_no() self.transfer_list[transfer_no] = {} self.transfer_list[transfer_no]["sn"] = send_sn send_data_len = len(upload_data) upload_ack_data = self.ack_pool.get() send_ack_len = len(upload_ack_data) magic = "P" pack_type = 2 if self.on_road_num > g.config.concurent_thread_num * 0.8: server_timeout = 0 else: server_timeout = g.config.roundtrip_timeout / 2 upload_data_head = struct.pack("<cBB8sIIBIH", magic, g.protocol_version, pack_type, str(self.session_id), transfer_no, send_sn, server_timeout, send_data_len, send_ack_len) upload_post_buf = base_container.WriteBuffer(upload_data_head) upload_post_buf.append(upload_data) upload_post_buf.append(upload_ack_data) upload_post_data = str(upload_post_buf) upload_post_data = encrypt_data(upload_post_data) try_no = 0 while self.running: try_no += 1 sleep_time = min(try_no, 30) self.last_roundtrip_time = time.time() start_time = time.time() with self.mutex: self.on_road_num += 1 # xlog.debug("start roundtrip transfer_no:%d send_data_len:%d ack_len:%d", transfer_no, send_data_len, send_ack_len) try: self.transfer_list[transfer_no]["try"] = try_no self.transfer_list[transfer_no]["stat"] = "request" self.transfer_list[transfer_no]["start"] = time.time() content, status, response = http_client.request(method="POST", path="data", data=upload_post_data, timeout=g.config.roundtrip_timeout) traffic = len(upload_post_data) + len(content) + 645 self.traffic += traffic g.quota -= traffic except Exception as e: xlog.exception("request except:%r retry %d", e, try_no) time.sleep(sleep_time) continue finally: with self.mutex: self.on_road_num -= 1 if status == 405: # session_id not exist on server if self.running: xlog.warn("server session_id not exist, start reset session") self.reset() return elif status == 200: recv_len = len(content) if recv_len < 6: xlog.error("roundtrip time:%d transfer_no:%d sn:%d send:%d status:%r retry:%d", (time.time() - start_time) * 1000, transfer_no, send_sn, send_data_len, len(content), status, try_no) continue content = decrypt_data(content) data = base_container.ReadBuffer(content) magic, version, pack_type = struct.unpack("<cBB", data.get(3)) if magic != "P" or version != g.protocol_version: xlog.error("get data head:%s", utils.str2hex(content[:2])) time.sleep(100) break if pack_type == 3: # error report error_code, message_len = struct.unpack("<BH", data.get(3)) message = data.get(message_len) xlog.warn("error report code:%d, msg:%s", error_code, message) if error_code == 1: # no quota xlog.warn("login x_server error:no quota") self.stop() return else: xlog.error("unknown error code:%d", error_code) return if pack_type != 2: # normal download traffic pack xlog.error("pack type:%d", pack_type) time.sleep(100) break sn, time_cost = struct.unpack("<II", data.get(8)) xlog.debug( "roundtrip time:%d cost:%d transfer_no:%d send_sn:%d send:%d recv_sn:%d rcv:%d status:%r", (time.time() - start_time) * 1000, time_cost, transfer_no, send_sn, send_data_len, sn, len(content), status) data_len = len(data) if (sn > 0 and data_len == 0) or (sn == 0 and data_len > 0): xlog.warn("get sn:%d len:%d %s", sn, data_len, data) if sn: self.last_download_data_time = time.time() last_roundtrip_download_size = data_len # xlog.debug("get sn:%d len:%d", sn, data_len) self.download_order_queue.put(sn, data) ack_pak = struct.pack("<Q", transfer_no) self.ack_pool.put(ack_pak) else: last_roundtrip_download_size = 0 if send_data_len == 0 and data_len > g.config.block_max_size: need_more_thread_num = int(g.config.concurent_thread_num * 0.5 - self.on_road_num) if need_more_thread_num > 0: for j in range(0, need_more_thread_num): if self.on_road_num > g.config.concurent_thread_num * 0.5: break self.touch_roundtrip() break else: xlog.warn("roundtrip time:%d transfer_no:%d send_sn:%d send:%d status:%r retry:%d", (time.time() - start_time) * 1000, transfer_no, send_sn, send_data_len, status, try_no) time.sleep(sleep_time) del self.transfer_list[transfer_no] xlog.info("roundtrip port:%d thread exit", server_address[1])