def run(self):
log.debug("pool thread start ")
while not self.stop_event.is_set():
_available_ips = self.get_all_available_ips()
sorted_keys = sorted(_available_ips)
with self.sorted_ips_lock:
self.sorted_ips = OrderedDict((key, _available_ips[key]) for key in sorted_keys)
self.stop_event.wait(self.wait_interval)
def call_api(self):
if self.lock:
with self.lock:
log.debug("sending thread lock api call")
result = self._call_api()
else:
result = self._call_api()
return result
def run(self):
while not self.stop_event.is_set():
self.stop_event.wait(self.heartbeat_interval)
if self.client and (time.time() - self.api.last_ack_time > self.heartbeat_interval):
try:
# 发送一个获取股票数量的包作为心跳包
self.api.do_heartbeat()
except Exception as e:
log.debug(str(e))
def perform_reflect(self, api_obj):
# ref : https://stackoverflow.com/questions/34439/finding-what-methods-an-object-has
method_names = [attr for attr in dir(api_obj) if callable(getattr(api_obj, attr))]
for method_name in method_names:
log.debug("testing attr %s" % method_name)
if method_name[:3] == 'get' or method_name == "do_heartbeat" or method_name == 'to_df':
log.debug("set refletion to method: %s", method_name)
_do_hp_api_call = partial(self.do_hq_api_call, method_name)
setattr(self, method_name, _do_hp_api_call)
def perform_reflect(self, api_obj):
# ref : https://stackoverflow.com/questions/34439/finding-what-methods-an-object-has
method_names = [attr for attr in dir(api_obj) if callable(getattr(api_obj, attr))]
for method_name in method_names:
log.debug("testing attr %s" % method_name)
if method_name[:3] == 'get' or method_name == "do_heartbeat" or method_name == 'to_df':
log.debug("set refletion to method: %s", method_name)
_do_hp_api_call = partial(self.do_hq_api_call, method_name)
setattr(self, method_name, _do_hp_api_call)
def run(self):
log.debug("pool thread start ")
while not self.stop_event.is_set():
_available_ips = self.get_all_available_ips()
sorted_keys = sorted(_available_ips)
with self.sorted_ips_lock:
self.sorted_ips = OrderedDict(
(key, _available_ips[key]) for key in sorted_keys)
self.stop_event.wait(self.wait_interval)
def wrapper(self, *args, **kw):
self.last_ack_time = time.time()
log.debug("last ack time update to " + str(self.last_ack_time))
try:
ret = func(self, *args, **kw)
except Exception as e:
self.last_transaction_failed = True
ret = None
raise e
finally:
return ret
def wrapper(self, *args, **kw):
self.last_ack_time = time.time()
log.debug("last ack time update to " + str(self.last_ack_time))
current_exception = None
try:
ret = func(self, *args, **kw)
except Exception as e:
current_exception = e
log.debug("hit exception on req exception is " + str(e))
if self.auto_retry:
for time_interval in self.retry_strategy.gen():
try:
time.sleep(time_interval)
self.disconnect()
self.connect(self.ip, self.port)
ret = func(self, *args, **kw)
if ret:
return ret
except Exception as retry_e:
current_exception = retry_e
log.debug("hit exception on *retry* req exception is " + str(retry_e))
log.debug("perform auto retry on req ")
self.last_transaction_failed = True
ret = None
if self.raise_exception:
to_raise = TdxFunctionCallError("calling function error")
to_raise.original_exception = current_exception if current_exception else None
raise to_raise
"""
如果raise_exception=True 抛出异常
如果raise_exception=False 返回None
"""
return ret
def __init__(self, hq_cls, ippool):
self.hq_cls = hq_cls
self.ippool = ippool
"""
正在通信的客户端连接
"""
self.api = hq_cls(multithread=True, heartbeat=True)
"""
备选连接
"""
self.hot_failover_api = hq_cls(multithread=True, heartbeat=True)
self.api_call_max_retry_times = DEFAULT_API_CALL_MAX_RETRY_TIMES
self.api_call_retry_times = 0
self.api_retry_interval = DEFAULT_API_RETRY_INTERVAL
# 对hq_cls 里面的get_系列函数进行反射
log.debug("perform_reflect")
self.perform_reflect(self.api)
def __init__(self, hq_cls, ippool):
self.hq_cls = hq_cls
self.ippool = ippool
"""
正在通信的客户端连接
"""
self.api = hq_cls(multithread=True, heartbeat=True)
"""
备选连接
"""
self.hot_failover_api = hq_cls(multithread=True, heartbeat=True)
self.api_call_max_retry_times = DEFAULT_API_CALL_MAX_RETRY_TIMES
self.api_call_retry_times = 0
self.api_retry_interval = DEFAULT_API_RETRY_INTERVAL
# 对hq_cls 里面的get_系列函数进行反射
log.debug("perform_reflect")
self.perform_reflect(self.api)
def call(self, func, params=None):
json_obj = {"func": func}
if params is not None:
json_obj["params"] = params
if self._transport_enc:
data_to_send = self.encrypt(json_obj)
response = self._session.post(self._endpoint, data=data_to_send)
else:
response = self._session.post(self._endpoint, json=json_obj)
response.encoding = self._encoding
text = response.text
if self._transport_enc:
decoded_text = self.decrypt(text)
log.debug(decoded_text)
return json.loads(decoded_text)
else:
return json.loads(text)
def get_all_available_ips(self):
"""
循环测试所有连接的连接速度和有效性
:return:
"""
_available_ips = OrderedDict()
for ip in self.ips:
ip_addr, port = ip
api = self.hq_class(multithread=False, heartbeat=False)
try:
with api.connect(ip_addr, port):
start_ts = time.time()
api.do_heartbeat()
end_ts = time.time()
diff_ts = end_ts - start_ts
_available_ips[diff_ts] = ip
log.debug("time diff is %f for %s" % (diff_ts, _available_ips))
except Exception as e:
log.debug("can not use %s:%d the exception is %s" % (ip_addr, port, str(e)))
continue
return _available_ips
def disconnect(self):
log.debug("primary api disconnected")
self.api.disconnect()
log.debug("hot backup api disconnected")
self.hot_failover_api.disconnect()
log.debug("ip pool released")
self.ippool.teardown()
def disconnect(self):
log.debug("primary api disconnected")
self.api.disconnect()
log.debug("hot backup api disconnected")
self.hot_failover_api.disconnect()
log.debug("ip pool released")
self.ippool.teardown()
def connect(self, ip='101.227.73.20', port=7709):
"""
:param ip: 服务器ip 地址
:param port: 服务器端口
:return: 是否连接成功 True/False
"""
self.client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.client.settimeout(CONNECT_TIMEOUT)
log.debug("connecting to server : %s on port :%d" % (ip, port))
try:
self.client.connect((ip, port))
except socket.timeout as e:
print(str(e))
log.debug("connection expired")
return False
log.debug("connected!")
if self.need_setup:
self.setup()
if self.heartbeat:
self.stop_event = threading.Event()
self.heartbeat_thread = HqHeartBeatThread(self, self.stop_event,
self.heartbeat_interval)
self.heartbeat_thread.start()
return self
def connect(self, ipandport, hot_failover_ipandport):
log.debug("setup ip pool")
self.ippool.setup()
log.debug("connecting to primary api")
self.api.connect(*ipandport)
log.debug("connecting to hot backup api")
self.hot_failover_api.connect(*hot_failover_ipandport)
return self
def connect(self, ipandport, hot_failover_ipandport):
log.debug("setup ip pool")
self.ippool.setup()
log.debug("connecting to primary api")
self.api.connect(*ipandport)
log.debug("connecting to hot backup api")
self.hot_failover_api.connect(*hot_failover_ipandport)
return self
def call(self, func, params=None):
json_obj = {
"func": func
}
if params is not None:
json_obj["params"] = params
if self._transport_enc:
data_to_send = self.encrypt(json_obj)
response = self._session.post(self._endpoint, data=data_to_send)
else:
response = self._session.post(self._endpoint, json=json_obj)
response.encoding = self._encoding
text = response.text
if self._transport_enc:
decoded_text = self.decrypt(text)
log.debug(decoded_text)
return json.loads(decoded_text)
else:
return json.loads(text)
def get_all_available_ips(self):
"""
循环测试所有连接的连接速度和有效性
:return:
"""
_available_ips = OrderedDict()
for ip in self.ips:
ip_addr, port = ip
api = self.hq_class(multithread=False, heartbeat=False)
try:
with api.connect(ip_addr, port):
start_ts = time.time()
api.do_heartbeat()
end_ts = time.time()
diff_ts = end_ts - start_ts
_available_ips[diff_ts] = ip
log.debug("time diff is %f for %s" %
(diff_ts, _available_ips))
except Exception as e:
log.debug("can not use %s:%d the exception is %s" %
(ip_addr, port, str(e)))
continue
return _available_ips
def disconnect(self):
if self.client:
log.debug("disconnecting")
try:
self.client.shutdown(socket.SHUT_RDWR)
self.client.close()
except Exception as e:
log.debug(str(e))
log.debug("disconnected")
def disconnect(self):
if self.heartbeat_thread and \
self.heartbeat_thread.is_alive():
self.stop_event.set()
if self.client:
log.debug("disconnecting")
try:
self.client.shutdown(socket.SHUT_RDWR)
self.client.close()
self.client = None
except Exception as e:
log.debug(str(e))
log.debug("disconnected")
def connect(self,
ip='101.227.73.20',
port=7709,
time_out=CONNECT_TIMEOUT,
bindport=None,
bindip='0.0.0.0'):
"""
:param ip: 服务器ip 地址
:param port: 服务器端口
:param time_out: 连接超时时间
:param bindport: 绑定的本地端口
:param bindip: 绑定的本地ip
:return: 是否连接成功 True/False
"""
self.client = TrafficStatSocket(socket.AF_INET, socket.SOCK_STREAM)
self.client.settimeout(time_out)
log.debug("connecting to server : %s on port :%d" % (ip, port))
try:
self.ip = ip
self.port = port
if bindport is not None:
self.client.bind((bindip, bindport))
self.client.connect((ip, port))
except socket.timeout as e:
# print(str(e))
log.debug("connection expired")
if self.raise_exception:
raise TdxConnectionError("connection timeout error")
return False
except Exception as e:
if self.raise_exception:
raise TdxConnectionError("other errors")
return False
log.debug("connected!")
if self.need_setup:
self.setup()
if self.heartbeat:
self.stop_event = threading.Event()
self.heartbeat_thread = HqHeartBeatThread(self, self.stop_event,
self.heartbeat_interval)
self.heartbeat_thread.start()
return self
def disconnect(self):
if self.heartbeat_thread and \
self.heartbeat_thread.is_alive():
self.stop_event.set()
if self.client:
log.debug("disconnecting")
try:
self.client.shutdown(socket.SHUT_RDWR)
self.client.close()
self.client = None
except Exception as e:
log.debug(str(e))
if self.raise_exception:
raise TdxConnectionError("disconnect err")
log.debug("disconnected")
def wrapper(self, *args, **kw):
self.last_ack_time = time.time()
log.debug("last ack time update to " + str(self.last_ack_time))
current_exception = None
try:
ret = func(self, *args, **kw)
except Exception as e:
current_exception = e
log.debug("hit exception on req exception is " + str(e))
if self.auto_retry:
for time_interval in self.retry_strategy.gen():
try:
time.sleep(time_interval)
self.disconnect()
self.connect(self.ip, self.port)
ret = func(self, *args, **kw)
if ret:
return ret
except Exception as retry_e:
current_exception = retry_e
log.debug(
"hit exception on *retry* req exception is " + str(retry_e))
log.debug("perform auto retry on req ")
self.last_transaction_failed = True
ret = None
if self.raise_exception:
to_raise = TdxFunctionCallError("calling function error")
to_raise.original_exception = current_exception if current_exception else None
raise to_raise
"""
如果raise_exception=True 抛出异常
如果raise_exception=False 返回None
"""
return ret
def connect(self, ip='101.227.73.20', port=7709, time_out=CONNECT_TIMEOUT, bindport=None, bindip='0.0.0.0'):
"""
:param ip: 服务器ip 地址
:param port: 服务器端口
:param time_out: 连接超时时间
:param bindport: 绑定的本地端口
:param bindip: 绑定的本地ip
:return: 是否连接成功 True/False
"""
self.client = TrafficStatSocket(socket.AF_INET, socket.SOCK_STREAM)
self.client.settimeout(time_out)
log.debug("connecting to server : %s on port :%d" % (ip, port))
try:
self.ip = ip
self.port = port
if bindport is not None:
self.client.bind((bindip, bindport))
self.client.connect((ip, port))
except socket.timeout as e:
# print(str(e))
log.debug("connection expired")
if self.raise_exception:
raise TdxConnectionError("connection timeout error")
return False
except Exception as e:
if self.raise_exception:
raise TdxConnectionError("other errors")
return False
log.debug("connected!")
if self.need_setup:
self.setup()
if self.heartbeat:
self.stop_event = threading.Event()
self.heartbeat_thread = HqHeartBeatThread(
self, self.stop_event, self.heartbeat_interval)
self.heartbeat_thread.start()
return self
def connect(self, ip, port):
"""
:param ip: 服务器ip 地址
:param port: 服务器端口
:return: 是否连接成功 True/False
"""
self.client = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.client.settimeout(CONNECT_TIMEOUT)
log.debug("connecting to server : %s on port :%d" % (ip, port))
try:
self.client.connect((ip, port))
except socket.timeout as e:
print(str(e))
log.debug("connection expired")
return False
log.debug("connected!")
if self.need_setup:
self.setup()
return self
def disconnect(self):
if self.client:
log.debug("disconnecting")
self.client.shutdown(socket.SHUT_RDWR)
self.client.close()
log.debug("disconnected")
if ip not in self.ips:
self.ips.append(ip)
if __name__ == "__main__":
from pytdx.hq import TdxHq_API
from pytdx.config.hosts import hq_hosts
import logging
log.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
# create formatter
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# add formatter to ch
ch.setFormatter(formatter)
log.addHandler(ch)
ips = [(v[1], v[2]) for v in hq_hosts]
pool = AvailableIPPool(TdxHq_API, ips)
pool.wait_interval = 60 * 5
pool.setup()
sleep_time = 130
log.debug("ready to sleep %d" % sleep_time)
time.sleep(sleep_time)
log.debug("sleep done")
ips = pool.get_ips()
log.debug(str(pool.get_ips()))
log.debug("ready to teardown")
pool.teardown()
if __name__ == "__main__":
from pytdx.hq import TdxHq_API
from pytdx.config.hosts import hq_hosts
import logging
log.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
# create formatter
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# add formatter to ch
ch.setFormatter(formatter)
log.addHandler(ch)
ips = [(v[1], v[2]) for v in hq_hosts]
pool = AvailableIPPool(TdxHq_API, ips)
pool.wait_interval = 60 * 5
pool.setup()
sleep_time = 130
log.debug("ready to sleep %d" % sleep_time )
time.sleep(sleep_time)
log.debug("sleep done")
ips = pool.get_ips()
log.debug(str(pool.get_ips()))
log.debug("ready to teardown")
pool.teardown()
def _call_api(self):
self.setup()
if not (self.client):
raise SocketClientNotReady("socket client not ready")
if not (self.send_pkg):
raise SendPkgNotReady("send pkg not ready")
nsended = self.client.send(self.send_pkg)
self.client.send_pkg_num += 1
self.client.send_pkg_bytes += nsended
self.client.last_api_send_bytes = nsended
if self.client.first_pkg_send_time is None:
self.client.first_pkg_send_time = datetime.datetime.now()
if DEBUG:
log.debug("send package:" + str(self.send_pkg))
if nsended != len(self.send_pkg):
log.debug("send bytes error")
raise SendRequestPkgFails("send fails")
else:
head_buf = self.client.recv(self.rsp_header_len)
if DEBUG:
log.debug("recv head_buf:" + str(head_buf) + " |len is :" +
str(len(head_buf)))
if len(head_buf) == self.rsp_header_len:
self.client.recv_pkg_num += 1
self.client.recv_pkg_bytes += self.rsp_header_len
_, _, _, zipsize, unzipsize = struct.unpack("<IIIHH", head_buf)
if DEBUG:
log.debug("zip size is: " + str(zipsize))
body_buf = bytearray()
last_api_recv_bytes = self.rsp_header_len
while True:
buf = self.client.recv(zipsize)
len_buf = len(buf)
self.client.recv_pkg_num += 1
self.client.recv_pkg_bytes += len_buf
last_api_recv_bytes += len_buf
body_buf.extend(buf)
if not (buf) or len_buf == 0 or len(body_buf) == zipsize:
break
self.client.last_api_recv_bytes = last_api_recv_bytes
if len(buf) == 0:
log.debug("接收数据体失败服务器断开连接")
raise ResponseRecvFails("接收数据体失败服务器断开连接")
if zipsize == unzipsize:
log.debug("不需要解压")
else:
log.debug("需要解压")
if sys.version_info[0] == 2:
unziped_data = zlib.decompress(buffer(body_buf))
else:
unziped_data = zlib.decompress(body_buf)
body_buf = unziped_data
## 解压
if DEBUG:
log.debug("recv body: ")
log.debug(body_buf)
return self.parseResponse(body_buf)
else:
log.debug("head_buf is not 0x10")
raise ResponseHeaderRecvFails("head_buf is not 0x10 : " +
str(head_buf))