Exemple #1
0
    def ack_process(self, ack):
        self.lock.acquire()
        try:
            last_ack = struct.unpack("<I", ack.get(4))[0]

            while len(ack):
                sn = struct.unpack("<I", ack.get(4))[0]
                # xlog.debug("ack: %d", sn)
                if sn in self.unacked_send_list:
                    self.unacked_send_list[sn] = "acked"

            for sn in self.unacked_send_list:
                if sn > last_ack:
                    continue
                if self.unacked_send_list[sn] == "acked":
                    continue

                # xlog.debug("last_ack:%d sn:%d", last_ack, sn)
                self.unacked_send_list[sn] = "acked"

            while (self.acked_send_continue_sn + 1) in self.unacked_send_list and \
                            self.unacked_send_list[self.acked_send_continue_sn + 1] == "acked":
                self.acked_send_continue_sn += 1
                del self.unacked_send_list[self.acked_send_continue_sn]

        except Exception as e:
            xlog.exception("ack_process:%r", e)
        finally:
            self.lock.release()
Exemple #2
0
    def ack_process(self, ack):
        self.lock.acquire()
        try:
            last_ack = struct.unpack("<I", ack.get(4))[0]

            while len(ack):
                sn = struct.unpack("<I", ack.get(4))[0]
                # xlog.debug("ack: %d", sn)
                if sn in self.wait_ack_send_list:
                    self.wait_ack_send_list[sn] = "acked"

            for sn in self.wait_ack_send_list:
                if sn > last_ack:
                    continue
                if self.wait_ack_send_list[sn] == "acked":
                    continue

                # xlog.debug("last_ack:%d sn:%d", last_ack, sn)
                self.wait_ack_send_list[sn] = "acked"

            while (self.ack_send_continue_sn + 1) in self.wait_ack_send_list and \
                    self.wait_ack_send_list[self.ack_send_continue_sn + 1] == "acked":
                self.ack_send_continue_sn += 1
                del self.wait_ack_send_list[self.ack_send_continue_sn]

        except Exception as e:
            xlog.exception("ack_process:%r", e)
        finally:
            self.lock.release()
Exemple #3
0
 def parse_body(self, data):
     self.body_len = len(data)
     self.max_age, self.port, protocol_id_length = struct.unpack("!LHxB", data[:8])
     pos = 8
     self.protocol_id = data[pos:pos+protocol_id_length].tobytes()
     pos += protocol_id_length
     host_length = struct.unpack("!B", data[pos:pos+1])[0]
     pos += 1
     self.host = data[pos:pos+host_length].tobytes()
     pos += host_length
     self.parse_origin(data[pos:])
Exemple #4
0
    def cmd_processor(self):
        while self.running:
            data = self.get_cmd_data()
            if not data:
                break

            cmd_id = struct.unpack("<B", data.get(1))[0]
            if cmd_id == 1:  # data
                self.send_to_sock(data)

            elif cmd_id == 3:  # ack:
                position = struct.unpack("<Q", data.get(8))[0]
                self.xlog.debug("Conn session:%s conn:%d ACK:%d", self.session.session_id, self.conn_id, position)
                if position > self.remote_acked_position:
                    self.remote_acked_position = position
                    self.recv_notice.acquire()
                    self.recv_notice.notify()
                    self.recv_notice.release()

            elif cmd_id == 2:  # Closed
                dat = data.get()
                if isinstance(dat, memoryview):
                    dat = dat.tobytes()
                self.xlog.debug("Conn session:%s conn:%d Peer Close:%s", self.session.session_id, self.conn_id, dat)
                if self.is_client:
                    self.transfer_peer_close("finish")
                self.stop("peer close")

            elif cmd_id == 0:  # Create connect
                if self.port or len(self.host) or self.next_cmd_seq != 1 or self.sock:
                    raise Exception("put_send_data %s conn:%d Create but host:%s port:%d next seq:%d" % (
                        self.session.session_id, self.conn_id,
                        self.host, self.port, self.next_cmd_seq))

                self.sock_type = struct.unpack("<B", data.get(1))[0]
                host_len = struct.unpack("<H", data.get(2))[0]
                self.host = data.get(host_len)
                self.port = struct.unpack("<H", data.get(2))[0]

                sock, res = self.do_connect(self.host, self.port)
                if res is False:
                    self.xlog.debug("Conn session:%s conn:%d %s:%d Create fail", self.session.session_id, self.conn_id,
                               self.host, self.port)
                    self.transfer_peer_close("connect fail")
                else:
                    self.xlog.info("Conn session:%s conn:%d %s:%d", self.session.session_id, self.conn_id, self.host,
                              self.port)
                    self.sock = sock
                    self.recv_thread = threading.Thread(target=self.recv_worker)
                    self.recv_thread.start()
            else:
                self.xlog.error("Conn session:%s conn:%d unknown cmd_id:%d",
                                self.session.session_id, self.conn_id, cmd_id)
                raise Exception("put_send_data unknown cmd_id:%d" % cmd_id)
Exemple #5
0
    def cmd_processor(self):
        while self.running:
            data = self.get_cmd_data()
            if not data:
                break

            cmd_id = struct.unpack("<B", data.get(1))[0]
            if cmd_id == 1:  # data
                self.send_to_sock(data)

            elif cmd_id == 3:  # ack:
                position = struct.unpack("<Q", data.get(8))[0]
                self.xlog.debug("Conn session:%s conn:%d ACK:%d", self.session.session_id, self.conn_id, position)
                if position > self.remote_acked_position:
                    self.remote_acked_position = position
                    self.recv_notice.acquire()
                    self.recv_notice.notify()
                    self.recv_notice.release()

            elif cmd_id == 2:  # Closed
                dat = data.get()
                if isinstance(dat, memoryview):
                    dat = dat.tobytes()
                self.xlog.debug("Conn session:%s conn:%d Peer Close:%s", self.session.session_id, self.conn_id, dat)
                if self.is_client:
                    self.transfer_peer_close("finish")
                self.stop("peer close")

            elif cmd_id == 0:  # Create connect
                if self.port or len(self.host) or self.next_cmd_seq != 1 or self.sock:
                    raise Exception("put_send_data %s conn:%d Create but host:%s port:%d next seq:%d" % (
                        self.session.session_id, self.conn_id,
                        self.host, self.port, self.next_cmd_seq))

                self.sock_type = struct.unpack("<B", data.get(1))[0]
                host_len = struct.unpack("<H", data.get(2))[0]
                self.host = data.get(host_len)
                self.port = struct.unpack("<H", data.get(2))[0]

                sock, res = self.do_connect(self.host, self.port)
                if res is False:
                    self.xlog.debug("Conn session:%s conn:%d %s:%d Create fail", self.session.session_id, self.conn_id,
                               self.host, self.port)
                    self.transfer_peer_close("connect fail")
                else:
                    self.xlog.info("Conn session:%s conn:%d %s:%d", self.session.session_id, self.conn_id, self.host,
                              self.port)
                    self.sock = sock
                    self.recv_thread = threading.Thread(target=self.recv_worker)
                    self.recv_thread.start()
            else:
                self.xlog.error("Conn session:%s conn:%d unknown cmd_id:%d",
                                self.session.session_id, self.conn_id, cmd_id)
                raise Exception("put_send_data unknown cmd_id:%d" % cmd_id)
Exemple #6
0
 def parse_body(self, data):
     self.body_len = len(data)
     self.max_age, self.port, protocol_id_length = struct.unpack(
         "!LHxB", data[:8])
     pos = 8
     self.protocol_id = data[pos:pos + protocol_id_length].tobytes()
     pos += protocol_id_length
     host_length = struct.unpack("!B", data[pos:pos + 1])[0]
     pos += 1
     self.host = data[pos:pos + host_length].tobytes()
     pos += host_length
     self.parse_origin(data[pos:])
Exemple #7
0
	def struct_callback(s):
		(name, id, description), s = unpack('SIS', s)

		if ObjectParamsStructDesc.has_key(id):
			output_extra = {}
			for substruct, ename, edescription in ObjectParamsStructDesc[id]:
				o, s = unpack(substruct, s)
				output_extra[ename] = o
			output = [name, id, description, output_extra]
		else:
			output = [name, id, description, {}]

		return output, s
Exemple #8
0
def unpack_response(response):
    try:
        data = response.task.read(size=2)
        if not data:
            raise GAE_Exception(600, "get protocol head fail")

        if len(data) !=2:
            raise GAE_Exception(600, "get protocol head fail, data:%s, len:%d" % (data, len(data)))

        headers_length, = struct.unpack('!h', data)
        data = response.task.read(size=headers_length)
        if not data:
            raise GAE_Exception(600,
                "get protocol head fail, len:%d" % headers_length)

        raw_response_line, headers_data = inflate(data).split('\r\n', 1)
        _, status, reason = raw_response_line.split(None, 2)
        response.app_status = int(status)
        response.app_reason = reason.strip()

        headers_block, app_msg = headers_data.split('\r\n\r\n')
        headers_pairs = headers_block.split('\r\n')
        response.headers = {}
        for pair in headers_pairs:
            if not pair:
                break
            k, v = pair.split(': ', 1)
            response.headers[k] = v

        response.app_msg = app_msg

        return response
    except Exception as e:
        response.worker.close("unpack protocol error")
        raise GAE_Exception(600, "unpack protocol:%r at:%s" % (e, traceback.format_exc()))
	def __process__(self, data, **kw):
		# Unpack the first lot of data
		args, leftover = unpack(self.struct, data)
		self.__init__(self.sequence, *args, **kw)

		if hasattr(self.__class__, "__process__"):
			self.__class__.__process__(self, leftover, **kw)
			return

		# Unpack the second lot of data
		try:
			moreargs, leftover2 = unpack(self.substruct, leftover)
			if len(leftover2) > 0:
				raise ValueError("\nError when processing %s.\nClass Structure: %s, Given Data: %r\nExtra data found: %r " % (self.__class__, self.substruct, leftover, leftover2))
		except TypeError, e:
			raise ValueError("\nError when processing %s.\nClass Structure: %s, Given Data: %r\nNot enough data found: %s" % (self.__class__, self.substruct, leftover, e))
Exemple #10
0
 def parse_priority_data(self, data):
     MASK = 0x80000000
     self.depends_on, self.stream_weight = struct.unpack(
         "!LB", data[:5]
     )
     self.exclusive = bool(self.depends_on & MASK)
     self.depends_on &= ~MASK
     return 5
Exemple #11
0
    def round_trip_process(self, data, ack):
        while len(data):
            sn, plen = struct.unpack("<II", data.get(8))
            pdata = data.get_buf(plen)
            # xlog.debug("download sn:%d len:%d", sn, plen)

            self.receive_process.put(sn, pdata)

        self.ack_process(ack)
    def fromstr(cls, data):
        """\
		Look at the packet type and morph this object into the
		correct type.
		"""
        args, extra = unpack(Header.struct, data)
        if len(extra) > 0:
            raise ValueError("Got too much data! %s bytes remaining" % len(extra))

        return cls(*args)
Exemple #13
0
    def roundtrip_process(self, data, ack):
        while len(data):
            sn, plen = struct.unpack("<II", data.get(8))
            pdata = data.get_buf(plen)
            # xlog.debug("upload sn:%d len:%d", sn, plen)

            self.receive_process.put(sn, pdata)
            self.last_upload_time = time.time()

        self.ack_process(ack)
Exemple #14
0
	def unpack(self, s):
		"""
		unpack() -> values, leftover
		
		Returns the unpacked values and any data that is left over.
		"""
		a, s = xstruct.unpack(self.xstruct, s)
		if len(a) != 1:
			raise TypeError("WTF?")
		return a[0], s
Exemple #15
0
    def download_data_processor(self, data):
        try:
            while len(data):
                conn_id, payload_len = struct.unpack("<II", data.get(8))
                payload = data.get_buf(payload_len)

                # xlog.debug("conn:%d upload data len:%d", conn_id, len(payload))
                if conn_id not in self.conn_list:
                    xlog.debug("conn:%d not exist", conn_id)
                    continue
                self.conn_list[conn_id].put_cmd_data(payload)
        except Exception as e:
            xlog.exception("download_data_processor:%r", e)
Exemple #16
0
    def download_data_processor(self, data):
        try:
            while len(data):
                conn_id, payload_len = struct.unpack("<II", data.get(8))
                payload = data.get_buf(payload_len)
                if conn_id not in self.conn_list:
                    #xlog.debug("DATA conn_id %d not in list", conn_id)
                    continue

                # xlog.debug("conn:%d upload data len:%d", conn_id, len(payload))
                self.conn_list[conn_id].put_cmd_data(payload)
        except Exception as e:
            xlog.exception("download_data_processor:%r", e)
Exemple #17
0
    def put_cmd_data(self, data):
        with self.cmd_notice:
            seq = struct.unpack("<I", data.get(4))[0]
            if seq < self.next_cmd_seq:
                # xlog.warn("put_send_data %s conn:%d seq:%d next:%d",
                #               self.session.session_id, self.conn_id,
                #               seq, self.next_cmd_seq)
                return

            self.cmd_queue[seq] = data.get_buf()

            if seq == self.next_cmd_seq:
                self.cmd_notice.notify()
Exemple #18
0
    def put_cmd_data(self, data):
        with self.cmd_notice:
            seq = struct.unpack("<I", data.get(4))[0]
            if seq < self.next_cmd_seq:
                # xlog.warn("put_send_data %s conn:%d seq:%d next:%d",
                #               self.session.session_id, self.conn_id,
                #               seq, self.next_cmd_seq)
                return

            self.cmd_queue[seq] = data.get_buf()

            if seq == self.next_cmd_seq:
                self.cmd_notice.notify()
Exemple #19
0
    def parse_frame_header(header):
        """
        Takes a 9-byte frame header and returns a tuple of the appropriate
        Frame object and the length that needs to be read from the socket.
        """
        fields = struct.unpack("!HBBBL", header)
        # First 24 bits are frame length.
        length = (fields[0] << 8) + fields[1]
        type = fields[2]
        flags = fields[3]
        stream_id = fields[4]

        if type not in FRAMES:
            raise ValueError("Unknown frame type %d" % type)

        frame = FRAMES[type](stream_id)
        frame.parse_flags(flags)
        return frame, length
Exemple #20
0
def parse_data(data):
    if len(data) == 0:
        return ""

    o = ""

    data = bytes(data)
    data = base_container.ReadBuffer(data)
    while len(data):

        sn, block_len = struct.unpack("<II", data.get(8))
        block = data.get_buf(block_len)

        o += "sn:%d {" % sn

        while len(block):
            conn_id, payload_len = struct.unpack("<II", block.get(8))

            o += "conn:%d [" % conn_id
            conn_data = block.get_buf(payload_len)

            seq = struct.unpack("<I", conn_data.get(4))[0]
            cmd_id = struct.unpack("<B", conn_data.get(1))[0]
            conn_payload = conn_data.get_buf()
            if cmd_id == 0:  # create connection
                sock_type = struct.unpack("<B", conn_payload.get(1))[0]
                host_len = struct.unpack("<H", conn_payload.get(2))[0]
                host = str(bytes(conn_payload.get(host_len)))
                port = struct.unpack("<H", conn_payload.get(2))[0]
                o += "%d|Connect:%s:%d" % (seq, host, port)
            elif cmd_id == 1:  # data
                o += "%d|D:%d" % (seq, len(conn_payload))
            elif cmd_id == 2:  # closed
                o += "%d|Closed:%s" % (seq, conn_payload)
            elif cmd_id == 3:  # ack
                position = struct.unpack("<Q", conn_payload.get())[0]
                o += "%d|Ack:%d" % (seq, position)

            o += "],"
        o += "},"

    return o
Exemple #21
0
    def parse_frame_header(header):
        """
        Takes a 9-byte frame header and returns a tuple of the appropriate
        Frame object and the length that needs to be read from the socket.
        """
        fields = struct.unpack("!HBBBL", header)
        # First 24 bits are frame length.
        length = (fields[0] << 8) + fields[1]
        type = fields[2]
        flags = fields[3]
        stream_id = fields[4]

        if type not in FRAMES:
            raise ValueError("Unknown frame type %d" % type)

        frame = FRAMES[type](stream_id)
        frame.parse_flags(flags)
        return frame, length
Exemple #22
0
 def parse_body(self, data):
     padding_data_length = self.parse_padding_data(data)
     self.promised_stream_id = struct.unpack("!L", data[padding_data_length:padding_data_length + 4])[0]
     self.data = data[padding_data_length + 4:].tobytes()
     self.body_len = len(data)
Exemple #23
0
    def login_session(self):
        if len(g.server_host) == 0:
            return False

        start_time = time.time()
        while time.time() - start_time < 30:
            try:
                magic = b"P"
                pack_type = 1
                upload_data_head = struct.pack("<cBB8sIHIIHH", magic, g.protocol_version, pack_type,
                                               bytes(self.session_id),
                                               g.config.max_payload, g.config.send_delay, g.config.windows_size,
                                               int(g.config.windows_ack), g.config.resend_timeout, g.config.ack_delay)
                upload_data_head += struct.pack("<H", len(g.config.login_account)) + bytes(g.config.login_account,
                                                                                           encoding='iso-8859-1')
                upload_data_head += struct.pack("<H", len(g.config.login_password)) + bytes(g.config.login_password,
                                                                                            encoding='iso-8859-1')

                upload_post_data = encrypt_data(upload_data_head)

                content, status, response = g.http_client.request(method="POST", host=g.server_host, path="/data",
                                                                  data=upload_post_data,
                                                                  timeout=g.config.network_timeout)

                time_cost = time.time() - start_time

                if status == 521:
                    g.last_api_error = "session server is down."
                    xlog.warn("login session server is down, try get new server.")
                    g.server_host = None
                    return False

                if status != 200:
                    g.last_api_error = "session server login fail:%r" % status
                    xlog.warn("login session fail, status:%r", status)
                    continue

                if len(content) < 6:
                    g.last_api_error = "session server protocol fail, login res len:%d" % len(content)
                    xlog.error("login data len:%d fail", len(content))
                    continue

                info = decrypt_data(content)
                magic, protocol_version, pack_type, res, message_len = struct.unpack("<cBBBH", info[:6])
                message = info[6:]
                if isinstance(message, memoryview):
                    message = message.tobytes()

                if magic != b"P" or protocol_version != g.protocol_version or pack_type != 1:
                    xlog.error("login_session time:%d head error:%s", 1000 * time_cost, utils.str2hex(info[:6]))
                    return False

                if res != 0:
                    g.last_api_error = "session server login fail, code:%d msg:%s" % (res, message)
                    xlog.warn("login_session time:%d fail, res:%d msg:%s", 1000 * time_cost, res, message)
                    return False

                g.last_api_error = ""
                xlog.info("login_session %s time:%d msg:%s", self.session_id, 1000 * time_cost, message)
                return True
            except Exception as e:
                xlog.exception("login_session e:%r", e)
                time.sleep(1)

        return False
Exemple #24
0
    def parse_body(self, data):
        self.last_stream_id, self.error_code = struct.unpack("!LL", data[:8])
        self.body_len = len(data)

        if len(data) > 8:
            self.additional_data = data[8:].tobytes()
Exemple #25
0
 def parse_body(self, data):
     self.window_increment = struct.unpack("!L", data)[0]
     self.body_len = len(data)
    def __process__(self, data):
        args, leftover = unpack(self.struct, data)
        if len(leftover) > 0:
            raise ValueError("Left over data found for %r: '%r'" % (self.__class__.__name__, leftover))

        self.__init__(self.sequence, *args)
Exemple #27
0
 def parse_body(self, data):
     padding_data_length = self.parse_padding_data(data)
     self.promised_stream_id = struct.unpack(
         "!L", data[padding_data_length:padding_data_length + 4])[0]
     self.data = data[padding_data_length + 4:].tobytes()
     self.body_len = len(data)
Exemple #28
0
    def normal_round_trip_worker(self, work_id):
        while self.running:
            data, ack = self.get_send_data(work_id)

            if not self.running:
                return

            send_data_len = len(data)
            send_ack_len = len(ack)
            transfer_no = self.get_transfer_no()
            # xlog.debug("trip:%d no:%d send data:%s", work_id, transfer_no, parse_data(data))

            magic = b"P"
            pack_type = 2

            if self.send_buffer.pool_size > g.config.max_payload or \
                    (self.send_buffer.pool_size and len(self.wait_queue.waiters) < g.config.min_on_road):
                server_timeout = 0
            elif work_id > g.config.concurent_thread_num * 0.9:
                server_timeout = 1
            elif work_id > g.config.concurent_thread_num * 0.7:
                server_timeout = 3
            else:
                server_timeout = g.config.roundtrip_timeout

            request_session_id = self.session_id
            upload_data_head = struct.pack("<cBB8sIBIH", magic, g.protocol_version, pack_type,
                                           bytes(self.session_id), transfer_no,
                                           server_timeout, send_data_len, send_ack_len)
            upload_post_buf = base_container.WriteBuffer(upload_data_head)
            upload_post_buf.append(data)
            upload_post_buf.append(ack)
            upload_post_data = upload_post_buf.to_bytes()
            upload_post_data = encrypt_data(upload_post_data)
            self.last_send_time = time.time()

            sleep_time = 1

            start_time = time.time()

            with self.lock:
                self.on_road_num += 1
                self.transfer_list[transfer_no] = {}
                self.transfer_list[transfer_no]["stat"] = "request"
                self.transfer_list[transfer_no]["start"] = start_time

            # xlog.debug("start trip transfer_no:%d send_data_len:%d ack_len:%d timeout:%d",
            #           transfer_no, send_data_len, send_ack_len, server_timeout)
            try:
                content, status, response = g.http_client.request(method="POST", host=g.server_host,
                                                                  path="/data?tid=%d" % transfer_no,
                                                                  data=upload_post_data,
                                                                  headers={
                                                                      "Content-Length": str(len(upload_post_data))},
                                                                  timeout=server_timeout + g.config.network_timeout)

                traffic = len(upload_post_data) + len(content) + 645
                self.traffic += traffic
                g.quota -= traffic
                if g.quota < 0:
                    g.quota = 0
            except Exception as e:
                xlog.exception("request except:%r ", e)

                time.sleep(sleep_time)
                continue
            finally:
                with self.lock:
                    self.on_road_num -= 1
                    try:
                        if transfer_no in self.transfer_list:
                            del self.transfer_list[transfer_no]
                    except:
                        pass

            g.stat["roundtrip_num"] += 1
            roundtrip_time = (time.time() - start_time)

            if status == 521:
                xlog.warn("X-tunnel server is down, try get new server.")
                g.server_host = None
                self.stop()
                login_process()
                return

            if status != 200:
                xlog.warn("roundtrip time:%f transfer_no:%d send:%d status:%r ",
                          roundtrip_time, transfer_no, send_data_len, status)
                time.sleep(sleep_time)
                continue

            recv_len = len(content)
            if recv_len < 6:
                xlog.warn("roundtrip time:%f transfer_no:%d send:%d recv:%d Head",
                          roundtrip_time, transfer_no, send_data_len, recv_len)
                continue

            content = decrypt_data(content)
            payload = base_container.ReadBuffer(content)

            magic, version, pack_type = struct.unpack("<cBB", payload.get(3))
            if magic != b"P" or version != g.protocol_version:
                xlog.warn("get data head:%s", utils.str2hex(content[:2]))
                time.sleep(sleep_time)
                continue

            if pack_type == 3:  # error report
                error_code, message_len = struct.unpack("<BH", payload.get(3))
                message = payload.get(message_len)
                # xlog.warn("report code:%d, msg:%s", error_code, message)
                if error_code == 1:
                    # no quota
                    xlog.warn("x_server error:no quota")
                    self.stop()
                    return
                elif error_code == 2:
                    # unpack error
                    xlog.warn("roundtrip time:%f transfer_no:%d send:%d recv:%d unpack_error:%s",
                              roundtrip_time, transfer_no, send_data_len, len(content), message)
                    continue
                elif error_code == 3:
                    # session not exist
                    if self.session_id == request_session_id:
                        xlog.warn("server session_id:%s not exist, reset session.", request_session_id)
                        self.reset()
                        return
                    else:
                        continue
                else:
                    xlog.error("unknown error code:%d, message:%s", error_code, message)
                    time.sleep(sleep_time)
                    continue

            if pack_type != 2:  # normal download traffic pack
                xlog.error("pack type:%d", pack_type)
                time.sleep(100)
                continue

            time_cost, server_send_pool_size, data_len, ack_len = struct.unpack("<IIIH", payload.get(14))
            xlog.debug(
                "trip:%d no:%d tc:%f cost:%f to:%d snd:%d rcv:%d s_pool:%d on_road:%d target:%d",
                work_id, transfer_no,
                roundtrip_time, time_cost / 1000.0, server_timeout,
                send_data_len, len(content), server_send_pool_size,
                self.on_road_num,
                self.target_on_roads)

            if len(self.conn_list) == 0:
                self.target_on_roads = 0
            elif len(content) >= g.config.max_payload:
                self.target_on_roads = \
                    min(g.config.concurent_thread_num - g.config.min_on_road, self.target_on_roads + 10)
            elif len(content) <= 21:
                self.target_on_roads = max(g.config.min_on_road, self.target_on_roads - 5)
            self.trigger_more()

            rtt = roundtrip_time * 1000 - time_cost
            rtt = max(100, rtt)
            speed = (send_data_len + len(content) + 400) / rtt
            response.worker.update_debug_data(rtt, send_data_len, len(content), speed)
            if rtt > 8000:
                xlog.debug("rtt:%d speed:%d trace:%s", rtt, speed, response.worker.get_trace())
                xlog.debug("task trace:%s", response.task.get_trace())
                g.stat["slow_roundtrip"] += 1

            try:
                data = payload.get_buf(data_len)
                ack = payload.get_buf(ack_len)
            except Exception as e:
                xlog.warn("trip:%d no:%d data not enough %r", work_id, transfer_no, e)
                continue

            # xlog.debug("trip:%d no:%d recv data:%s", work_id, transfer_no, parse_data(data))

            try:
                self.round_trip_process(data, ack)

                self.last_receive_time = time.time()
            except Exception as e:
                xlog.exception("data process:%r", e)

        xlog.info("roundtrip thread exit")
Exemple #29
0
    def login_session(self):
        if len(g.server_host) == 0:
            return False

        start_time = time.time()
        while time.time() - start_time < 30:
            try:
                start_time = time.time()

                magic = "P"
                pack_type = 1
                upload_data_head = struct.pack("<cBB8sIHIIHH", magic, g.protocol_version, pack_type, str(self.session_id),
                                               g.config.max_payload, g.config.send_delay, g.config.windows_size,
                                               g.config.windows_ack, g.config.resend_timeout, g.config.ack_delay)
                upload_data_head += struct.pack("<H", len(g.config.login_account)) + str(g.config.login_account)
                upload_data_head += struct.pack("<H", len(g.config.login_password)) + str(g.config.login_password)

                upload_post_data = encrypt_data(upload_data_head)

                content, status, response = g.http_client.request(method="POST", host=g.server_host, path="/data",
                                                               data=upload_post_data,
                                                               timeout=g.config.network_timeout)

                time_cost = time.time() - start_time

                if status == 521:
                    g.last_api_error = "session server is down."
                    xlog.warn("login session server is down, try get new server.")
                    g.server_host = None
                    return False

                if status != 200:
                    g.last_api_error = "session server login fail:%r" % status
                    xlog.warn("login session fail, status:%r", status)
                    continue

                if len(content) < 6:
                    g.last_api_error = "session server protocol fail, login res len:%d" % len(content)
                    xlog.error("login data len:%d fail", len(content))
                    continue

                info = decrypt_data(content)
                magic, protocol_version, pack_type, res, message_len = struct.unpack("<cBBBH", info[:6])
                message = info[6:]
                if isinstance(message, memoryview):
                    message = message.tobytes()

                if magic != "P" or protocol_version != g.protocol_version or pack_type != 1:
                    xlog.error("login_session time:%d head error:%s", 1000 * time_cost, utils.str2hex(info[:6]))
                    return False

                if res != 0:
                    g.last_api_error = "session server login fail, code:%d msg:%s" % (res, message)
                    xlog.warn("login_session time:%d fail, res:%d msg:%s", 1000 * time_cost, res, message)
                    return False

                g.last_api_error = ""
                xlog.info("login_session %s time:%d msg:%s", self.session_id, 1000 * time_cost, message)
                return True
            except Exception as e:
                xlog.exception("login_session e:%r", e)
                time.sleep(1)

        return False
Exemple #30
0
    def normal_roundtrip_worker(self):
        while self.running:
            data, ack = self.get_send_data()
            if not self.running:
                return

            send_data_len = len(data)
            send_ack_len = len(ack)
            transfer_no = self.get_transfer_no()

            magic = "P"
            pack_type = 2

            if self.on_road_num > g.config.concurent_thread_num * 0.6:
                server_timeout = 0
            else:
                server_timeout = g.config.roundtrip_timeout

            request_session_id = self.session_id
            upload_data_head = struct.pack("<cBB8sIBIH", magic, g.protocol_version, pack_type,
                                           str(self.session_id), transfer_no,
                                           server_timeout, send_data_len, send_ack_len)
            upload_post_buf = base_container.WriteBuffer(upload_data_head)
            upload_post_buf.append(data)
            upload_post_buf.append(ack)
            upload_post_data = str(upload_post_buf)
            upload_post_data = encrypt_data(upload_post_data)
            self.last_send_time = time.time()

            sleep_time = 1

            start_time = time.time()

            with self.lock:
                self.on_road_num += 1
                self.transfer_list[transfer_no] = {}
                self.transfer_list[transfer_no]["stat"] = "request"
                self.transfer_list[transfer_no]["start"] = start_time

            #xlog.debug("start roundtrip transfer_no:%d send_data_len:%d ack_len:%d timeout:%d",
            #           transfer_no, send_data_len, send_ack_len, server_timeout)
            try:
                content, status, response = g.http_client.request(method="POST", host=g.server_host,
                                                                  path="/data?tid=%d" % transfer_no,
                                                                  data=upload_post_data,
                                                                  headers={"Content-Length": str(len(upload_post_data))},
                                                                timeout=server_timeout + g.config.network_timeout)

                traffic = len(upload_post_data) + len(content) + 645
                self.traffic += traffic
                g.quota -= traffic
            except Exception as e:
                xlog.exception("request except:%r ", e)

                time.sleep(sleep_time)
                continue
            finally:
                with self.lock:
                    self.on_road_num -= 1
                    try:
                        if transfer_no in self.transfer_list:
                            del self.transfer_list[transfer_no]
                    except:
                        pass

            g.stat["roundtrip_num"] += 1
            roundtrip_time = (time.time() - start_time) * 1000

            if status == 521:
                xlog.warn("X-tunnel server is down, try get new server.")
                g.server_host = None
                self.stop()
                login_process()
                return

            if status != 200:
                xlog.warn("roundtrip time:%d transfer_no:%d send:%d status:%r ",
                          roundtrip_time, transfer_no, send_data_len, status)
                time.sleep(sleep_time)
                continue

            recv_len = len(content)
            if recv_len < 6:
                xlog.warn("roundtrip time:%d transfer_no:%d send:%d recv:%d Head",
                          roundtrip_time, transfer_no, send_data_len, recv_len)
                continue

            content = decrypt_data(content)
            payload = base_container.ReadBuffer(content)

            magic, version, pack_type = struct.unpack("<cBB", payload.get(3))
            if magic != "P" or version != g.protocol_version:
                xlog.warn("get data head:%s", utils.str2hex(content[:2]))
                time.sleep(sleep_time)
                continue

            if pack_type == 3:  # error report
                error_code, message_len = struct.unpack("<BH", payload.get(3))
                message = payload.get(message_len)
                # xlog.warn("report code:%d, msg:%s", error_code, message)
                if error_code == 1:
                    # no quota
                    xlog.warn("x_server error:no quota")
                    self.stop()
                    return
                elif error_code == 2:
                    # unpack error
                    xlog.warn("roundtrip time:%d transfer_no:%d send:%d recv:%d unpack_error:%s",
                              roundtrip_time, transfer_no, send_data_len, len(content), message)
                    continue
                elif error_code == 3:
                    # session not exist
                    if self.session_id == request_session_id:
                        xlog.warn("server session_id:%s not exist, reset session.", request_session_id)
                        self.reset()
                        return
                    else:
                        continue
                else:
                    xlog.error("unknown error code:%d, message:%s", error_code, message)
                    time.sleep(sleep_time)
                    continue

            if pack_type != 2:  # normal download traffic pack
                xlog.error("pack type:%d", pack_type)
                time.sleep(100)
                continue

            time_cost, server_send_pool_size, data_len, ack_len = struct.unpack("<IIIH", payload.get(14))
            xlog.debug("roundtrip time:%d cost:%d transfer_no:%d send:%d rcv:%d ",
                roundtrip_time, time_cost, transfer_no, send_data_len, len(content))

            self.trigger_more(server_send_pool_size)

            rtt = roundtrip_time - time_cost
            rtt = max(100, rtt)
            speed = (send_data_len + len(content) + 400) / rtt
            response.worker.update_debug_data(rtt, send_data_len, len(content), speed)
            if rtt > 8000:
                xlog.debug("rtt:%d speed:%d trace:%s", rtt, speed, response.worker.get_trace())
                xlog.debug("task trace:%s", response.task.get_trace())
                g.stat["slow_roundtrip"] += 1

            try:
                data = payload.get_buf(data_len)
                ack = payload.get_buf(ack_len)
            except:
                xlog.debug("data not enough")
                continue

            try:
                self.roundtrip_process(data, ack)

                self.last_receive_time = time.time()
            except Exception as e:
                xlog.exception("data process:%r", e)
                continue

        xlog.info("roundtrip thread exit")
Exemple #31
0
 def parse_padding_data(self, data):
     if b'PADDED' in self.flags:
         self.pad_length = struct.unpack('!B', data[:1])[0]
         return 1
     return 0
Exemple #32
0
 def parse_body(self, data):
     self.window_increment = struct.unpack("!L", data)[0]
     self.body_len = len(data)
Exemple #33
0
    def parse_body(self, data):
        self.last_stream_id, self.error_code = struct.unpack("!LL", data[:8])
        self.body_len = len(data)

        if len(data) > 8:
            self.additional_data = data[8:].tobytes()
Exemple #34
0
    def parse_body(self, data):
        for i in range(0, len(data), 6):
            name, value = struct.unpack("!HL", data[i:i+6])
            self.settings[name] = value

        self.body_len = len(data)
Exemple #35
0
	def unpack(self, s):
		a, s = xstruct.unpack(self.xstruct, s)
		if len(a) != 1:
			raise TypeError("WTF?")
		return a[0], s
Exemple #36
0
    def parse_body(self, data):
        if len(data) != 4:
            raise ValueError()

        self.error_code = struct.unpack("!L", data)[0]
        self.body_len = len(data)
Exemple #37
0
	def __process__(self, data):
		try:
			args, leftover = unpack(self.struct, data, callback=self.struct_callback)
		except TypeError, e:
			raise TypeError("Problem when trying to unpack %s (%s) %s" % (self.__class__, self.struct, e))
Exemple #38
0
    def parse_body(self, data):
        if len(data) != 4:
            raise ValueError()

        self.error_code = struct.unpack("!L", data)[0]
        self.body_len = len(data)
Exemple #39
0
 def parse_priority_data(self, data):
     MASK = 0x80000000
     self.depends_on, self.stream_weight = struct.unpack("!LB", data[:5])
     self.exclusive = bool(self.depends_on & MASK)
     self.depends_on &= ~MASK
     return 5
Exemple #40
0
    def parse_body(self, data):
        for i in range(0, len(data), 6):
            name, value = struct.unpack("!HL", data[i:i + 6])
            self.settings[name] = value

        self.body_len = len(data)
Exemple #41
0
 def parse_padding_data(self, data):
     if 'PADDED' in self.flags:
         self.pad_length = struct.unpack('!B', data[:1])[0]
         return 1
     return 0