def get_stock_count():
api = TdxHq_API()
if api.connect('119.147.212.81', 7709):
print(api.get_security_count(0)) #参数为市场代码
print(api.get_security_list(0, 0)) #第一个参数为市场代码，第二个参数为起始位置
print(api.get_security_count(1))
print(api.to_df(api.get_security_list(1, 0)))
api.disconnect()
def _test_speed(self, ip, port=7709):
api = TdxHq_API(raise_exception=True, auto_retry=False)
_time = datetime.datetime.now()
# print(self.timeout)
try:
with api.connect(ip, port, time_out=1):
res = api.get_security_list(0, 1)
# print(res)
# print(len(res))
if len(api.get_security_list(0, 1)) > 800:
return (datetime.datetime.now() - _time).total_seconds()
else:
return datetime.timedelta(9, 9, 0).total_seconds()
except Exception as e:
return datetime.timedelta(9, 9, 0).total_seconds()
def QA_fetch_get_stock_list(type_='stock', ip=best_ip, port=7709):
api = TdxHq_API()
with api.connect(ip, port):
data = pd.concat([pd.concat([api.to_df(api.get_security_list(j, i * 1000)).assign(sse='sz' if j == 0 else 'sh').set_index(
['code', 'sse'], drop=False) for i in range(int(api.get_security_count(j) / 1000) + 1)], axis=0) for j in range(2)], axis=0)
if type_ in ['stock', 'gp']:
return pd.concat([data[data['sse'] == 'sz'][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 10000 <= 30][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 100000 != 2],
data[data['sse'] == 'sh'][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 100000 == 6]]).assign(code=data['code'].apply(lambda x: str(x)))
#.assign(szm=data['name'].apply(lambda x: ''.join([y[0] for y in lazy_pinyin(x)])))\
#.assign(quanpin=data['name'].apply(lambda x: ''.join(lazy_pinyin(x))))
elif type_ in ['index', 'zs']:
return pd.concat([data[data['sse'] == 'sz'][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 1000 >= 399],
data[data['sse'] == 'sh'][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 1000 == 0]]) \
.sort_index()\
.assign(code=data['code'].apply(lambda x: str(x)))
#.assign(szm=data['name'].apply(lambda x: ''.join([y[0] for y in lazy_pinyin(x)])))\
#.assign(quanpin=data['name'].apply(lambda x: ''.join(lazy_pinyin(x))))
elif type_ in ['etf', 'ETF']:
return pd.concat([data[data['sse'] == 'sz'][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 10000 == 15],
data[data['sse'] == 'sh'][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 10000 == 51]]).sort_index().assign(code=data['code'].apply(lambda x: str(x)))\
#.assign(szm=data['name'].apply(lambda x: ''.join([y[0] for y in lazy_pinyin(x)])))\
#.assign(quanpin=data['name'].apply(lambda x: ''.join(lazy_pinyin(x))))
else:
return data.assign(code=data['code'].apply(lambda x: str(x)))
def _ping(self, ip, port=7709, first=False):
print('TDX{}: Start connecting to {}:{}...'.format(
self._tdxNo, ip, port))
tdxApi = TdxHq_API(raise_exception=True)
startTime = datetime.datetime.now()
try:
tdxApi.connect(ip, port, time_out=1)
if len(tdxApi.get_security_list(0, 1)) > 800:
print('TDX{}: Connect to {}:{} successfully'.format(
self._tdxNo, ip, port))
return tdxApi, datetime.datetime.now() - startTime
print('TDX{}: Bad response from {}:{}'.format(
self._tdxNo, ip, port))
except TypeError as ex:
print('TDX{}: Exception happened when ping {}:{}, {}'.format(
self._tdxNo, ip, port, ex))
if first:
self._info.print(
'TDX{}: pytdx版本错误，请先pip uninstall pytdx，然后再pip install pytdx'
.format(self._tdxNo), DyLogData.error)
except Exception as ex:
print('TDX{}: Exception happened when ping {}:{}, {}'.format(
self._tdxNo, ip, port, ex))
return None, None
def __hq_ping(self, ip, port):
api = TdxHq_API()
with api.connect(ip, port, time_out=0.7):
assert len(api.get_security_list(0, 1)) > 800
api.disconnect()
return True
return False
def sync_data():
"""将股票列表更新到数据库"""
host = SETTINGS.get('MONGO_HOST', "localhost")
port = SETTINGS.get('MONGO_PORT', 27017)
ms = MongoDBService(host, port)
ms.connect_db()
collection = ms.db_client["stocks"]["security"]
# 模拟交易flask配置参数
api = TdxHq_API()
with api.connect(SETTINGS["TDX_HOST"], SETTINGS["TDX_PORT"]):
for i in range(0, 2):
n = 0
batch_list = []
while True:
data = api.get_security_list(i, n * 1000)
if not len(data):
logging.warning(f"market[{i}] finished")
break
logging.warning(f"[{n*1000}-{(n+1)*1000}] write to db")
batch_list.extend([
UpdateOne(
{
"code": x.get("code"),
"market": str(i)
},
{"$set": dict(x)},
upsert=True,
) for x in data
])
n += 1
if batch_list:
collection.bulk_write(batch_list, ordered=False)
def QA_fetch_get_stock_list(type_='stock', ip=best_ip['stock'], port=7709):
api = TdxHq_API()
with api.connect(ip, port):
data = pd.concat([pd.concat([api.to_df(api.get_security_list(j, i * 1000)).assign(sse='sz' if j == 0 else 'sh').set_index(
['code', 'sse'], drop=False) for i in range(int(api.get_security_count(j) / 1000) + 1)], axis=0) for j in range(2)], axis=0)
#data.code = data.code.apply(int)
sz = data.query('sse=="sz"')
sh = data.query('sse=="sh"')
sz = sz.assign(sec=sz.code.apply(for_sz))
sh = sh.assign(sec=sh.code.apply(for_sh))
if type_ in ['stock', 'gp']:
return pd.concat([sz, sh]).query('sec=="stock_cn"').sort_index().assign(name=data['name'].apply(lambda x: str(x)[0:6]))
elif type_ in ['index', 'zs']:
return pd.concat([sz, sh]).query('sec=="index_cn"').sort_index().assign(name=data['name'].apply(lambda x: str(x)[0:6]))
#.assign(szm=data['name'].apply(lambda x: ''.join([y[0] for y in lazy_pinyin(x)])))\
#.assign(quanpin=data['name'].apply(lambda x: ''.join(lazy_pinyin(x))))
elif type_ in ['etf', 'ETF']:
return pd.concat([sz, sh]).query('sec=="etf_cn"').sort_index().assign(name=data['name'].apply(lambda x: str(x)[0:6]))
else:
return data.assign(code=data['code'].apply(lambda x: str(x))).assign(name=data['name'].apply(lambda x: str(x)[0:6]))
def _test_speed(self, ip, port=7709):
api = TdxHq_API(raise_exception=True, auto_retry=False)
_time = datetime.datetime.now()
# print(self.timeout)
try:
with api.connect(ip, port, time_out=1):
res = api.get_security_list(0, 1)
# print(res)
# print(len(res))
if len(api.get_security_list(0, 1)) > 800:
return (datetime.datetime.now() - _time).total_seconds()
else:
return datetime.timedelta(9, 9, 0).total_seconds()
except Exception as e:
return datetime.timedelta(9, 9, 0).total_seconds()
def _codes():
api = TdxHq_API()
with api.connect(ip):
rst = api.get_security_list(0, 255)
return rst
def ping(ip):
__time1 = datetime.datetime.now()
api = TdxHq_API()
try:
with api.connect(ip, 7709):
if len(api.get_security_list(0, 1)) > 800:
return datetime.datetime.now() - __time1
except:
return datetime.timedelta(9, 9, 0)
def ping(ip):
api = TdxHq_API()
__time1 = datetime.datetime.now()
try:
with api.connect(ip, 7709):
if len(api.get_security_list(0, 1)) > 800:
return datetime.datetime.now() - __time1
except:
print('Bad REPSONSE %s' % ip)
return datetime.timedelta(9, 9, 0)
def ping(ip):
api = TdxHq_API()
__time1 = datetime.datetime.now()
try:
with api.connect(ip, 7709):
if len(api.get_security_list(0, 1)) > 800:
return datetime.datetime.now() - __time1
except:
print('Bad REPSONSE %s' % ip)
return datetime.timedelta(9, 9, 0)
def ping(ip, port=7709, type_='stock'):
api = TdxHq_API()
apix = TdxExHq_API()
__time1 = datetime.datetime.now()
try:
if type_ in ['stock']:
with api.connect(ip, port, time_out=0.9):
res = api.get_security_list(0, 1)
if res is not None:
if len(api.get_security_list(0, 1)) > 800:
return datetime.datetime.now() - __time1
else:
print('BAD RESPONSE {}'.format(ip))
return __badtime
else:
print('BAD RESPONSE {}'.format(ip))
return __badtime
elif type_ in ['future']:
with apix.connect(ip, port, time_out=0.7):
res = apix.get_instrument_count()
if res is not None:
if res > 20000:
return datetime.datetime.now() - __time1
else:
print(f'️Bad FUTUREIP REPSONSE {ip}')
return __badtime
else:
print('️Bad FUTUREIP REPSONSE {}'.format(ip))
return __badtime
except Exception as e:
if isinstance(e, TypeError):
print(e)
print(ip, port)
print('Tushare内置的pytdx版本和QUANTAXIS使用的pytdx 版本不同, 请重新安装pytdx以解决此问题')
print('pip uninstall pytdx')
print('pip install pytdx')
else:
print('BAD RESPONSE {}'.format(ip))
return __badtime
def ping(self, ip, port):
api = TdxHq_API()
time1 = datetime.datetime.now()
try:
with api.connect(ip, int(port)):
if len(api.get_security_list(0, 1)) > 800:
return datetime.datetime.now() - time1
else:
return datetime.timedelta(9, 9, 0)
except:
return datetime.timedelta(9, 9, 0)
def _test_speed(self, ip, port=7709):
api = TdxHq_API(raise_exception=True, auto_retry=False)
#api.need_setup = False
_time = datetime.datetime.now()
try:
with api.connect(ip, port, time_out=0.05):
if len(api.get_security_list(0, 1)) > 800:
return (datetime.datetime.now() - _time).total_seconds()
else:
return datetime.timedelta(9, 9, 0).total_seconds()
except Exception as e:
#print('BAD IP {}, DEL for Reason{}'.format(ip,e))
return datetime.timedelta(9, 9, 0).total_seconds()
def __hq_list(self, market):
assert self.__hq.qsize() > 0
api = TdxHq_API()
ip, port = self.__hq.get()
with api.connect(ip, port):
df = list()
for start in range(0, api.get_security_count(market=market), 1000):
df.append(api.to_df(api.get_security_list(market, start)))
api.disconnect()
self.__hq.put((ip, port))
df = pandas.concat(df, sort=False).assign(sse=market)
df = df[[
'code', 'volunit', 'decimal_point', 'name', 'pre_close', 'sse'
]].dropna()
df = df.assign(sse='sh' if market == CODE_MARKET_SH else 'sz',
sec=get_code_type(df.code.tolist(), market))
return df
return None
def _ping(self, ip, port=7709):
print('TDX{}: Start connecting to {}:{}...'.format(
self._tdxNo, ip, port))
tdxApi = TdxHq_API(raise_exception=True)
startTime = datetime.datetime.now()
try:
tdxApi.connect(ip, port, time_out=1)
if len(tdxApi.get_security_list(0, 1)) > 800:
print('TDX{}: Connect to {}:{} successfully'.format(
self._tdxNo, ip, port))
return tdxApi, datetime.datetime.now() - startTime
print('TDX{}: Bad response from {}:{}'.format(
self._tdxNo, ip, port))
except Exception as ex:
print('TDX{}: Exception happened when ping {}:{}, {}'.format(
self._tdxNo, ip, port, ex))
return None, None
def QA_fetch_get_stock_list(type_='stock', ip=best_ip, port=7709):
api = TdxHq_API()
with api.connect(ip, port):
data = pd.concat([pd.concat([api.to_df(api.get_security_list(j, i * 1000)).assign(sse='sz' if j == 0 else 'sh').set_index(
['code', 'sse'], drop=False) for i in range(int(api.get_security_count(j) / 1000) + 1)], axis=0) for j in range(2)], axis=0)
if type_ in ['stock', 'gp']:
return pd.concat([data[data['sse'] == 'sz'][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 10000 <= 30][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 100000 != 2],
data[data['sse'] == 'sh'][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 100000 == 6]]).assign(code=data['code'].apply(lambda x: str(x)))
elif type_ in ['index', 'zs']:
return pd.concat([data[data['sse'] == 'sz'][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 1000 >= 399],
data[data['sse'] == 'sh'][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 1000 == 0]]).sort_index().assign(code=data['code'].apply(lambda x: str(x)))
elif type_ in ['etf', 'ETF']:
return pd.concat([data[data['sse'] == 'sz'][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 10000 == 15],
data[data['sse'] == 'sh'][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 10000 == 51]]).sort_index().assign(code=data['code'].apply(lambda x: str(x)))
else:
return data.assign(code=data['code'].apply(lambda x: str(x)))
class tdxApi:
def __init__(self):
self.ipset = [(v[1], v[2]) for v in hq_hosts]
random.shuffle(self.ipset)
self.ippool = AvailableIPPool(TdxHq_API, self.ipset)
self.primary_ip, _ = self.ippool.sync_get_top_n(2)
self.api = TdxHq_API(multithread=True, heartbeat=True, auto_retry=True)
def connect(self):
self.api.connect(self.primary_ip[0], self.primary_ip[1])
def release(self):
self.api.disconnect()
def getQuotes(self, stock_list):
datas = self.api.get_security_quotes(stock_list)
if datas is None:
return None
else:
return self.api.to_df(datas)
def getList(self, market, index):
df = self.api.to_df(self.api.get_security_list(market, index))
return df
def getCount(self, market=0):
return self.api.get_security_count(market)
def getMinQuotes(self, market, stock_code):
datas = self.api.get_minute_time_data(market, stock_code)
if datas is None:
return None
return self.api.to_df(datas)
def getMinQuotesHis(self, market, stock_code, date):
datas = self.api.get_history_minute_time_data(market, stock_code, date)
if datas is None:
return None
return self.api.to_df(datas)
def __do_fetch_security_list(self, api: TdxHq_API, exchange: Exchange):
arr = []
start = 0
ex_nbr = map_exchange_to_tdx_number[exchange]
count = api.get_security_count(ex_nbr)
while start < count:
tmp = api.get_security_list(ex_nbr, start)
if tmp is None:
break
sz = len(tmp)
if sz < 1:
break
start += sz
arr.extend(tmp)
print('total fetched rows: {}'.format(start))
df = api.to_df(arr).assign(ex=exchange.name)
df = df.assign(
stype=df.code.apply(exchange.code_to_type_str)
).set_index(['code', 'ex'], drop=True)
return df[df.stype.notnull()]
def ping(ip, port=7709, type_='stock'):
api = TdxHq_API()
apix = TdxExHq_API()
__time1 = datetime.datetime.now()
try:
if type_ in ['stock']:
with api.connect(ip, port, time_out=0.7):
if len(api.get_security_list(0, 1)) > 800:
return datetime.datetime.now() - __time1
else:
print('BAD RESPONSE {}'.format(ip))
return datetime.timedelta(9, 9, 0)
elif type_ in ['future']:
with apix.connect(ip, port, time_out=0.7):
if apix.get_instrument_count() > 10000:
return datetime.datetime.now() - __time1
else:
print('️Bad FUTUREIP REPSONSE {}'.format(ip))
return datetime.timedelta(9, 9, 0)
except:
print('BAD RESPONSE {}'.format(ip))
return datetime.timedelta(9, 9, 0)
def ping(ip, type_='stock'):
api = TdxHq_API()
apix = TdxExHq_API()
__time1 = datetime.datetime.now()
try:
if type_ in ['stock']:
with api.connect(ip, 7709, time_out=0.7):
if len(api.get_security_list(0, 1)) > 800:
return datetime.datetime.now() - __time1
else:
print('Bad STOCKIP REPSONSE %s' % ip)
return datetime.timedelta(9, 9, 0)
elif type_ in ['future']:
with apix.connect(ip, 7727, time_out=0.7):
if apix.get_instrument_count() > 40000:
return datetime.datetime.now() - __time1
else:
print('Bad FUTUREIP REPSONSE %s' % ip)
return datetime.timedelta(9, 9, 0)
except:
print('Bad REPSONSE %s' % ip)
return datetime.timedelta(9, 9, 0)
def QA_fetch_get_index_list(ip=None, port=None):
"""获取指数列表
Keyword Arguments:
ip {[type]} -- [description] (default: {None})
port {[type]} -- [description] (default: {None})
Returns:
[type] -- [description]
"""
ip, port = get_mainmarket_ip(ip, port)
api = TdxHq_API()
with api.connect(ip, port):
data = pd.concat([pd.concat([api.to_df(api.get_security_list(j, i * 1000)).assign(sse='sz' if j == 0 else 'sh').set_index(
['code', 'sse'], drop=False) for i in range(int(api.get_security_count(j) / 1000) + 1)], axis=0) for j in range(2)], axis=0)
#data.code = data.code.apply(int)
sz = data.query('sse=="sz"')
sh = data.query('sse=="sh"')
sz = sz.assign(sec=sz.code.apply(for_sz))
sh = sh.assign(sec=sh.code.apply(for_sh))
return pd.concat([sz, sh]).query('sec=="index_cn"').sort_index().assign(name=data['name'].apply(lambda x: str(x)[0:6]))
def QA_fetch_get_stock_list(type_='stock', ip=None, port=None):
global best_ip
if ip is None and port is None and best_ip['stock']['ip'] is None and best_ip['stock']['port'] is None:
best_ip = select_best_ip()
ip = best_ip['stock']['ip']
port = best_ip['stock']['port']
elif ip is None and port is None and best_ip['stock']['ip'] is not None and best_ip['stock']['port'] is not None:
ip = best_ip['stock']['ip']
port = best_ip['stock']['port']
else:
pass
api = TdxHq_API()
with api.connect(ip, port):
data = pd.concat([pd.concat([api.to_df(api.get_security_list(j, i * 1000)).assign(sse='sz' if j == 0 else 'sh').set_index(
['code', 'sse'], drop=False) for i in range(int(api.get_security_count(j) / 1000) + 1)], axis=0) for j in range(2)], axis=0)
#data.code = data.code.apply(int)
sz = data.query('sse=="sz"')
sh = data.query('sse=="sh"')
sz = sz.assign(sec=sz.code.apply(for_sz))
sh = sh.assign(sec=sh.code.apply(for_sh))
if type_ in ['stock', 'gp']:
return pd.concat([sz, sh]).query('sec=="stock_cn"').sort_index().assign(name=data['name'].apply(lambda x: str(x)[0:6]))
elif type_ in ['index', 'zs']:
return pd.concat([sz, sh]).query('sec=="index_cn"').sort_index().assign(name=data['name'].apply(lambda x: str(x)[0:6]))
#.assign(szm=data['name'].apply(lambda x: ''.join([y[0] for y in lazy_pinyin(x)])))\
#.assign(quanpin=data['name'].apply(lambda x: ''.join(lazy_pinyin(x))))
elif type_ in ['etf', 'ETF']:
return pd.concat([sz, sh]).query('sec=="etf_cn"').sort_index().assign(name=data['name'].apply(lambda x: str(x)[0:6]))
else:
return data.assign(code=data['code'].apply(lambda x: str(x))).assign(name=data['name'].apply(lambda x: str(x)[0:6]))
def single_client_benchmark(ip):
def _log(msg):
click.echo("HQ_BENCHMARK: [{:15s}] {} ".format(ip, datetime.datetime.now()) + msg)
def _grouped_list(stocks):
return [stocks[i:i + GET_QUOTES_PER_GROUPS] for i in range(0, len(stocks), GET_QUOTES_PER_GROUPS)]
_log("start benchmark")
total_time = connecting_time = get_security_count_time = get_security_list_time = get_security_quotes_time = num = 0
start_time = time.time()
last_time = start_time
try:
api = TdxHq_API(multithread=True)
port = 7709
if ":" in ip:
ip, port = ip.split(':')
port = int(port)
with api.connect(ip=ip, port=port):
_log("connected")
cur_time = time.time()
connecting_time = cur_time - last_time
last_time = cur_time
_log("connecting time is {}".format(connecting_time))
num = api.get_security_count(0)
_log("all shenzhen market stock count is {}".format(num))
cur_time = time.time()
get_security_count_time = cur_time - last_time
last_time = cur_time
_log("get_security_count_time is {}".format(get_security_count_time))
all = []
for i in range((num // 1000) + 1):
offset = i * 1000
section = api.get_security_list(0, offset)
all = all + section
cur_time = time.time()
get_security_list_time = cur_time - last_time
last_time = cur_time
_log("get_security_list_time is {}".format(get_security_list_time))
codes = [one['code'] for one in all]
results = []
for stocks in _grouped_list(codes):
req_list = [(0, code) for code in stocks]
one_results = api.get_security_quotes(req_list)
results = results + one_results
cur_time = time.time()
get_security_quotes_time = cur_time - last_time
last_time = cur_time
_log("get_security_quotes_time is {}".format(get_security_quotes_time))
total_time = last_time - start_time
_log("total_time is {}".format(total_time))
_log("end benchmark")
except Exception as e:
_log("hit exception " + str(e))
return {
"ip": ip,
"total_time": total_time,
"connecting_time": connecting_time,
"get_security_count_time": get_security_count_time,
"get_security_list_time": get_security_list_time,
"get_security_quotes_time": get_security_quotes_time,
"security_count": num
}
from pytdx.hq import TdxHq_API
api = TdxHq_API()
with api.connect('119.147.212.81', 7709):
df = api.to_df(api.get_security_quotes([(0, '000001'), (1, '600300')]))
print(df.to_json())
data = api.get_minute_time_data(1, '600300')
print(data)
list_df = api.to_df(api.get_security_list(0, 300))
print(list_df.to_json())
def test_all_functions(multithread, heartbeat, auto_retry, raise_exception):
api = TdxHq_API(multithread=multithread, heartbeat=heartbeat,
auto_retry=auto_retry, raise_exception=raise_exception)
with api.connect(time_out=30):
log.info("获取股票行情")
stocks = api.get_security_quotes([(0, "000001"), (1, "600300")])
assert stocks is not None
assert type(stocks) is list
# 方法2
stocks = api.get_security_quotes(0, "000001")
assert stocks is not None
assert type(stocks) is list
# 方法3
stocks = api.get_security_quotes((0, "000001"))
assert stocks is not None
assert type(stocks) is list
log.info("获取k线")
data = api.get_security_bars(9, 0, '000001', 4, 3)
assert data is not None
assert type(data) is list
assert len(data) == 3
log.info("获取 深市 股票数量")
assert api.get_security_count(0) > 0
log.info("获取股票列表")
stocks = api.get_security_list(1, 0)
assert stocks is not None
assert type(stocks) is list
assert len(stocks) > 0
log.info("获取指数k线")
data = api.get_index_bars(9, 1, '000001', 1, 2)
assert data is not None
assert type(data) is list
assert len(data) == 2
log.info("查询分时行情")
data = api.get_minute_time_data(TDXParams.MARKET_SH, '600300')
assert data is not None
log.info("查询历史分时行情")
data = api.get_history_minute_time_data(
TDXParams.MARKET_SH, '600300', 20161209)
assert data is not None
assert type(data) is list
assert len(data) > 0
log.info("查询分时成交")
data = api.get_transaction_data(TDXParams.MARKET_SZ, '000001', 0, 30)
assert data is not None
assert type(data) is list
log.info("查询历史分时成交")
data = api.get_history_transaction_data(
TDXParams.MARKET_SZ, '000001', 0, 10, 20170209)
assert data is not None
assert type(data) is list
assert len(data) == 10
log.info("查询公司信息目录")
data = api.get_company_info_category(TDXParams.MARKET_SZ, '000001')
assert data is not None
assert type(data) is list
assert len(data) > 0
start = data[0]['start']
length = data[0]['length']
log.info("读取公司信息-最新提示")
data = api.get_company_info_content(
0, '000001', '000001.txt', start, length)
assert data is not None
assert len(data) > 0
log.info("读取除权除息信息")
data = api.get_xdxr_info(1, '600300')
assert data is not None
assert type(data) is list
assert len(data) > 0
log.info("读取财务信息")
data = api.get_finance_info(0, '000001')
assert data is not None
assert type(data) is OrderedDict
assert len(data) > 0
log.info("日线级别k线获取函数")
data = api.get_k_data('000001', '2017-07-01', '2017-07-10')
assert type(data) is pd.DataFrame
assert len(data) == 6
log.info("获取板块信息")
data = api.get_and_parse_block_info(TDXParams.BLOCK_FG)
assert data is not None
assert type(data) is list
assert len(data) > 0
class Engine:
def __init__(self, *args, **kwargs):
if kwargs.pop('best_ip', False):
self.ip = self.best_ip
else:
self.ip = '14.17.75.71'
self.ip = kwargs.pop('ip', '14.17.75.71')
self.thread_num = kwargs.pop('thread_num', 1)
if not PY2 and self.thread_num != 1:
self.use_concurrent = True
else:
self.use_concurrent = False
self.api = TdxHq_API(args, kwargs)
if self.use_concurrent:
self.apis = [
TdxHq_API(args, kwargs) for i in range(self.thread_num)
]
self.executor = ThreadPoolExecutor(self.thread_num)
def connect(self):
self.api.connect(self.ip)
if self.use_concurrent:
for api in self.apis:
api.connect(self.ip)
return self
def __enter__(self):
return self
def exit(self):
self.api.disconnect()
if self.use_concurrent:
for api in self.apis:
api.disconnect()
def __exit__(self, exc_type, exc_val, exc_tb):
self.api.disconnect()
if self.use_concurrent:
for api in self.apis:
api.disconnect()
def quotes(self, code):
code = [code] if not isinstance(code, list) else code
code = self.security_list[self.security_list.code.isin(
code)].index.tolist()
data = [
self.api.to_df(
self.api.get_security_quotes(code[80 * pos:80 * (pos + 1)]))
for pos in range(int(len(code) / 80) + 1)
]
return pd.concat(data)
# data = data[['code', 'open', 'high', 'low', 'price']]
# data['datetime'] = datetime.datetime.now()
# return data.set_index('code', drop=False, inplace=False)
def stock_quotes(self):
code = self.stock_list.index.tolist()
if self.use_concurrent:
res = {
self.executor.submit(self.apis[pos % self.thread_num].get_security_quotes,
code[80 * pos:80 * (pos + 1)]) \
for pos in range(int(len(code) / 80) + 1)}
return pd.concat([self.api.to_df(dic.result()) for dic in res])
else:
data = [
self.api.to_df(
self.api.get_security_quotes(code[80 * pos:80 *
(pos + 1)]))
for pos in range(int(len(code) / 80) + 1)
]
return pd.concat(data)
@lazyval
def security_list(self):
return pd.concat([
pd.concat([
self.api.to_df(self.api.get_security_list(
j, i * 1000)).assign(sse=0 if j == 0 else 1).set_index(
['sse', 'code'], drop=False)
for i in range(int(self.api.get_security_count(j) / 1000) + 1)
],
axis=0) for j in range(2)
],
axis=0)
@lazyval
def stock_list(self):
aa = map(stock_filter, self.security_list.index.tolist())
return self.security_list[list(aa)]
@lazyval
def best_ip(self):
return select_best_ip()
@lazyval
def concept(self):
return self.api.to_df(
self.api.get_and_parse_block_info(TDXParams.BLOCK_GN))
@lazyval
def index(self):
return self.api.to_df(
self.api.get_and_parse_block_info(TDXParams.BLOCK_SZ))
@lazyval
def fengge(self):
return self.api.to_df(
self.api.get_and_parse_block_info(TDXParams.BLOCK_FG))
@lazyval
def customer_block(self):
return CustomerBlockReader().get_df(CUSTOMER_BLOCK_PATH)
@lazyval
def gbbq(self):
df = GbbqReader().get_df(GBBQ_PATH).query('category == 1')
df['datetime'] = pd.to_datetime(df['datetime'], format='%Y%m%d')
return df
def get_security_type(self, code):
if code in self.security_list.code.values:
return self.security_list[self.security_list.code ==
code]['sse'].as_matrix()[0]
else:
raise SecurityNotExists()
def get_security_bars(self, code, freq, index=False):
if index:
exchange = self.get_security_type(code)
func = self.api.get_index_bars
else:
exchange = get_stock_type(code)
func = self.api.get_security_bars
df = pd.DataFrame()
if freq in ['1d', 'day']:
freq = 9
elif freq in ['1m', 'min']:
freq = 8
else:
raise Exception("1d and 1m frequency supported only")
res = []
start = 0
while True:
data = func(freq, exchange, code, start, 800)
if not data:
break
res = data + res
start += 800
df = self.api.to_df(res).drop(
['year', 'month', 'day', 'hour', 'minute'], axis=1)
df['datetime'] = pd.to_datetime(df.datetime)
df['code'] = code
return df.set_index('datetime')
def _get_transaction(self, code, date):
res = []
start = 0
while True:
data = self.api.get_history_transaction_data(
get_stock_type(code), code, start, 2000, date)
if not data:
break
start += 2000
res = data + res
if len(res) == 0:
return pd.DataFrame()
df = self.api.to_df(res).assign(date=date)
df.index = pd.to_datetime(str(date) + " " + df["time"])
df['code'] = code
return df.drop("time", axis=1)
def time_and_price(self, code):
start = 0
res = []
exchange = self.get_security_type(code)
while True:
data = self.api.get_transaction_data(exchange, code, start, 2000)
if not data:
break
res = data + res
start += 2000
df = self.api.to_df(res)
df.time = pd.to_datetime(
str(pd.to_datetime('today').date()) + " " + df['time'])
df.loc[0, 'time'] = df.time[1]
return df.set_index('time')
@classmethod
def minute_bars_from_transaction(cls, transaction, freq):
if transaction.empty:
return pd.DataFrame()
data = transaction['price'].resample(freq,
label='right',
closed='left').ohlc()
data['volume'] = transaction['vol'].resample(freq,
label='right',
closed='left').sum()
data['code'] = transaction['code'][0]
return fillna(data)
def get_k_data(self, code, start, end, freq):
if isinstance(start, str) or isinstance(end, str):
start = pd.Timestamp(start)
end = pd.Timestamp(end)
sessions = pd.date_range(start, end)
trade_days = map(int, sessions.strftime("%Y%m%d"))
if freq == '1m':
freq = '1 min'
if freq == '1d':
freq = '24 H'
res = []
for trade_day in trade_days:
df = Engine.minute_bars_from_transaction(
self._get_transaction(code, trade_day), freq)
if df.empty:
continue
res.append(df)
if len(res) != 0:
return pd.concat(res)
return pd.DataFrame()
class Engine:
concurrent_thread_count = 50
def __init__(self, *args, **kwargs):
if 'ip' in kwargs:
self.ip = kwargs.pop('ip')
else:
if kwargs.pop('best_ip', False):
self.ip = self.best_ip
else:
self.ip = '14.17.75.71'
if 'concurrent_thread_count' in kwargs:
self.concurrent_thread_count = kwargs.pop(
'concurrent_thread_count', 50)
self.thread_num = kwargs.pop('thread_num', 1)
self.api = TdxHq_API(args, kwargs, raise_exception=True)
def connect(self):
self.api.connect(self.ip)
return self
def __enter__(self):
return self
def exit(self):
self.api.disconnect()
def __exit__(self, exc_type, exc_val, exc_tb):
self.api.disconnect()
def quotes(self, code):
code = [code] if not isinstance(code, list) else code
code = self.security_list[self.security_list.code.isin(
code)].index.tolist()
data = [
self.api.to_df(
self.api.get_security_quotes(code[80 * pos:80 * (pos + 1)]))
for pos in range(int(len(code) / 80) + 1)
]
return pd.concat(data)
# data = data[['code', 'open', 'high', 'low', 'price']]
# data['datetime'] = datetime.datetime.now()
# return data.set_index('code', drop=False, inplace=False)
def stock_quotes(self):
code = self.stock_list.index.tolist()
data = [
self.api.to_df(
self.api.get_security_quotes(code[80 * pos:80 * (pos + 1)]))
for pos in range(int(len(code) / 80) + 1)
]
return pd.concat(data)
@lazyval
def security_list(self):
return pd.concat([
pd.concat([
self.api.to_df(self.api.get_security_list(
j, i * 1000)).assign(sse=0 if j == 0 else 1).set_index(
['sse', 'code'], drop=False)
for i in range(int(self.api.get_security_count(j) / 1000) + 1)
],
axis=0) for j in range(2)
],
axis=0)
@lazyval
def stock_list(self):
aa = map(stock_filter, self.security_list.index.tolist())
return self.security_list[list(aa)]
@lazyval
def best_ip(self):
return select_best_ip()
@lazyval
def concept(self):
return self.api.to_df(
self.api.get_and_parse_block_info(TDXParams.BLOCK_GN))
@lazyval
def index(self):
return self.api.to_df(
self.api.get_and_parse_block_info(TDXParams.BLOCK_SZ))
@lazyval
def fengge(self):
return self.api.to_df(
self.api.get_and_parse_block_info(TDXParams.BLOCK_FG))
@lazyval
def block(self):
return self.api.to_df(
self.api.get_and_parse_block_info(TDXParams.BLOCK_DEFAULT))
@lazyval
def customer_block(self):
return CustomerBlockReader().get_df(CUSTOMER_BLOCK_PATH)
def xdxr(self, code):
df = self.api.to_df(
self.api.get_xdxr_info(self.get_security_type(code), code))
if df.empty:
return df
df['datetime'] = pd.to_datetime((df.year * 10000 + df.month * 100 +
df.day).apply(lambda x: str(x)))
return df.drop(['year', 'month', 'day'], axis=1).set_index('datetime')
@lazyval
def gbbq(self):
df = GbbqReader().get_df(GBBQ_PATH).query('category == 1')
df['datetime'] = pd.to_datetime(df['datetime'], format='%Y%m%d')
return df
def get_security_type(self, code):
if code in self.security_list.code.values:
return self.security_list[self.security_list.code ==
code]['sse'].as_matrix()[0]
else:
raise SecurityNotExists()
@retry(3)
def get_security_bars(self, code, freq, start=None, end=None, index=False):
if index:
exchange = self.get_security_type(code)
func = self.api.get_index_bars
else:
exchange = get_stock_type(code)
func = self.api.get_security_bars
if start:
start = start.tz_localize(None)
if end:
end = end.tz_localize(None)
if freq in ['1d', 'day']:
freq = 9
elif freq in ['1m', 'min']:
freq = 8
else:
raise Exception("1d and 1m frequency supported only")
res = []
pos = 0
while True:
data = func(freq, exchange, code, pos, 800)
if not data:
break
res = data + res
pos += 800
if start and pd.to_datetime(data[0]['datetime']) < start:
break
try:
df = self.api.to_df(res).drop(
['year', 'month', 'day', 'hour', 'minute'], axis=1)
df['datetime'] = pd.to_datetime(df.datetime)
df.set_index('datetime', inplace=True)
if freq == 9:
df.index = df.index.normalize()
except ValueError: # 未上市股票，无数据
logger.warning("no k line data for {}".format(code))
# return pd.DataFrame({
# 'amount': [0],
# 'close': [0],
# 'open': [0],
# 'high': [0],
# 'low': [0],
# 'vol': [0],
# 'code': code
# },
# index=[start]
# )
return pd.DataFrame()
close = [df.close.values[-1]]
if start:
df = df.loc[lambda df: start <= df.index]
if end:
df = df.loc[lambda df: df.index.normalize() <= end]
if df.empty:
# return pd.DataFrame({
# 'amount': [0],
# 'close': close,
# 'open': close,
# 'high': close,
# 'low': close,
# 'vol': [0],
# 'code': code
# },
# index=[start]
# )
return df
else:
if int(df['vol'][-1]) <= 0 and end == df.index[-1] and len(
df) == 1: # 成交量为0，当天返回的是没开盘的数据
return pd.DataFrame()
df['code'] = code
return df
def _get_transaction(self, code, date):
res = []
start = 0
while True:
data = self.api.get_history_transaction_data(
get_stock_type(code), code, start, 2000, date)
if not data:
break
start += 2000
res = data + res
if len(res) == 0:
return pd.DataFrame()
df = self.api.to_df(res).assign(date=date)
df.loc[0, 'time'] = df.time[1]
df.index = pd.to_datetime(str(date) + " " + df["time"])
df['code'] = code
return df.drop("time", axis=1)
def time_and_price(self, code):
start = 0
res = []
exchange = self.get_security_type(code)
while True:
data = self.api.get_transaction_data(exchange, code, start, 2000)
if not data:
break
res = data + res
start += 2000
df = self.api.to_df(res)
df.time = pd.to_datetime(
str(pd.to_datetime('today').date()) + " " + df['time'])
df.loc[0, 'time'] = df.time[1]
return df.set_index('time')
@classmethod
def minute_bars_from_transaction(cls, transaction, freq):
if transaction.empty:
return pd.DataFrame()
mask = transaction.index < transaction.index[0].normalize(
) + pd.Timedelta('12 H')
def resample(transaction):
if transaction.empty:
return pd.DataFrame()
data = transaction['price'].resample(freq,
label='right',
closed='left').ohlc()
data['volume'] = transaction['vol'].resample(freq,
label='right',
closed='left').sum()
data['code'] = transaction['code'][0]
return data
morning = resample(transaction[mask])
afternoon = resample(transaction[~mask])
if morning.empty and afternoon.empty:
return pd.DataFrame()
if not afternoon.empty:
morning.index.values[-1] = afternoon.index[0] - pd.Timedelta(
'1 min')
df = pd.concat([morning, afternoon])
return fillna(df)
def _get_k_data(self, code, freq, sessions):
trade_days = map(int, sessions.strftime("%Y%m%d"))
if freq == '1m':
freq = '1 min'
if freq == '1d':
freq = '24 H'
res = []
concurrent_count = self.concurrent_thread_count
jobs = []
for trade_day in trade_days:
# df = Engine.minute_bars_from_transaction(self._get_transaction(code, trade_day), freq)
reqevent = gevent.spawn(Engine.minute_bars_from_transaction,
self._get_transaction(code, trade_day),
freq)
jobs.append(reqevent)
if len(jobs) >= concurrent_count:
gevent.joinall(jobs, timeout=30)
for j in jobs:
if j.value is not None and not j.value.empty:
res.append(j.value)
jobs.clear()
gevent.joinall(jobs, timeout=30)
for j in jobs:
if j.value is not None and not j.value.empty:
res.append(j.value)
jobs.clear()
if len(res) != 0:
return pd.concat(res)
return pd.DataFrame()
def get_k_data(self, code, start, end, freq, check=True):
if isinstance(start, str) or isinstance(end, str):
start = pd.Timestamp(start)
end = pd.Timestamp(end)
if check:
daily_bars = self.get_security_bars(code, '1d', start, end)
if daily_bars is None or daily_bars.empty:
return daily_bars
sessions = daily_bars.index
else:
sessions = pd.bdate_range(start,
end,
weekmask='Mon Tue Wed Thu Fri')
df = self._get_k_data(code, freq, sessions)
def check_df(freq, df, daily_bars):
if freq == '1m':
need_check = pd.DataFrame({
'open':
df['open'].resample('1D').first(),
'high':
df['high'].resample('1D').max(),
'low':
df['low'].resample('1D').min(),
'close':
df['close'].resample('1D').last(),
'volume':
df['volume'].resample('1D').sum()
}).dropna()
else:
need_check = df
if daily_bars.shape[0] != need_check.shape[0]:
logger.warning("{} merged {}, expected {}".format(
code, need_check.shape[0], daily_bars.shape[0]))
need_check = fillna(
need_check.reindex(daily_bars.index, copy=False))
diff = daily_bars[['open',
'close']] == need_check[['open', 'close']]
res = (diff.open) & (diff.close)
sessions = res[res == False].index
return sessions
if not df.empty:
if check:
sessions = check_df(freq, df, daily_bars)
if sessions.shape[0] != 0:
logger.info(
"fixing data for {}-{} with sessions: {}".format(
code, freq, sessions))
fix = self._get_k_data(code, freq, sessions)
df.loc[fix.index] = fix
return df
return df
class StdQuotes(object):
"""股票市场实时行情"""
bestip = ('47.103.48.45', 7709)
def __init__(self, **kwargs):
try:
default = settings.get('SERVER').get('HQ')[0]
self.bestip = config.get('BESTIP').get('HQ', default)
except ValueError:
self.config = None
self.client = TdxHq_API(**kwargs)
def traffic(self):
with self.client.connect(*self.bestip):
return self.client.get_traffic_stats()
def quotes(self, symbol=[]):
'''
获取实时日行情数据
:param symbol: 股票代码
:return: pd.dataFrame or None
'''
logger.debug(type(logger))
if type(symbol) is str:
symbol = [symbol]
with self.client.connect(*self.bestip):
symbol = get_stock_markets(symbol)
result = self.client.get_security_quotes(symbol)
return to_data(result)
def bars(self, symbol='000001', frequency='9', start='0', offset='100'):
'''
获取实时日K线数据
:param symbol: 股票代码
:param frequency: 数据类别
:param market: 证券市场
:param start: 开始位置
:param offset: 每次获取条数
:return: pd.dataFrame or None
'''
with self.client.connect(*self.bestip):
market = get_stock_market(symbol)
result = self.client.get_security_bars(int(frequency), int(market),
str(symbol), int(start),
int(offset))
return to_data(result)
def stock_count(self, market=MARKET_SH):
'''
获取市场股票数量
:param market: 股票市场代码 sh 上海， sz 深圳
:return: pd.dataFrame or None
'''
with self.client.connect(*self.bestip):
result = self.client.get_security_count(market=market)
return result
def stocks(self, market=MARKET_SH):
'''
获取股票列表
:param market:
:return:
'''
with self.client.connect(*self.bestip):
counts = self.client.get_security_count(market=market)
stocks = None
for start in tqdm(range(0, counts, 1000)):
result = self.client.get_security_list(market=market,
start=start)
stocks = pandas.concat(
[stocks, to_data(result)],
ignore_index=True) if start > 1 else to_data(result)
return stocks
def index_bars(self,
symbol='000001',
frequency='9',
start='0',
offset='100'):
'''
获取指数k线
:param symbol:
:param frequency:
:param start:
:param offset:
:return:
'''
with self.client.connect(*self.bestip):
market = get_stock_market(symbol)
result = self.client.get_index_bars(frequency=frequency,
market=market,
code=symbol,
start=start,
count=offset)
return to_data(result)
def minute(self, symbol=''):
'''
获取实时分时数据
:param market: 证券市场
:param symbol: 股票代码
:return: pd.DataFrame
'''
with self.client.connect(*self.bestip):
market = get_stock_market(symbol)
result = self.client.get_minute_time_data(market=market,
code=symbol)
return to_data(result)
def minutes(self, symbol='', date='20191023'):
'''
分时历史数据
:param market:
:param symbol:
:param date:
:return: pd.dataFrame or None
'''
with self.client.connect(*self.bestip):
market = get_stock_market(symbol)
result = self.client.get_history_minute_time_data(market=market,
code=symbol,
date=date)
return to_data(result)
def transaction(self, symbol='', start=0, offset=10):
'''
查询分笔成交
:param market: 市场代码
:param symbol: 股票代码
:param start: 起始位置
:param offset: 请求数量
:return: pd.dataFrame or None
'''
with self.client.connect(*self.bestip):
market = get_stock_market(symbol)
result = self.client.get_transaction_data(int(market), symbol,
int(start), int(offset))
return to_data(result)
def transactions(self, symbol='', start=0, offset=10, date='20170209'):
'''
查询历史分笔成交
参数：市场代码， 股票代码，起始位置，日期 数量 如： 0,000001,0,10,20170209
:param symbol: 股票代码
:param start: 起始位置
:param offset: 数量
:param date: 日期
:return: pd.dataFrame or None
'''
with self.client.connect(*self.bestip):
market = get_stock_market(symbol, string=False)
result = self.client.get_history_transaction_data(market=market,
code=symbol,
start=start,
count=offset,
date=date)
return to_data(result)
def F10C(self, symbol=''):
'''
查询公司信息目录
:param market: 市场代码
:param symbol: 股票代码
:return: pd.dataFrame or None
'''
with self.client.connect(*self.bestip):
market = get_stock_market(symbol)
result = self.client.get_company_info_category(int(market), symbol)
return result
def F10(self, symbol='', name=''):
'''
读取公司信息详情
:param name: 公司 F10 标题
:param symbol: 股票代码
:return: pd.dataFrame or None
'''
with self.client.connect(*self.bestip):
result = {}
market = get_stock_market(symbol, string=False)
frequency = self.client.get_company_info_category(
int(market), symbol)
if name:
for x in frequency:
if x['name'] == name:
return self.client.get_company_info_content(
market=market,
code=symbol,
filename=x['filename'],
start=x['start'],
length=x['length'])
for x in frequency:
result[x['name']] = self.client.get_company_info_content(
market=market,
code=symbol,
filename=x['filename'],
start=x['start'],
length=x['length'])
else:
pass
return result
def xdxr(self, symbol=''):
'''
读取除权除息信息
:param market: 市场代码
:param symbol: 股票代码
:return: pd.dataFrame or None
'''
with self.client.connect(*self.bestip):
market = get_stock_market(symbol)
result = self.client.get_xdxr_info(int(market), symbol)
return to_data(result)
def finance(self, symbol='000001'):
'''
读取财务信息
:param symbol:
:return:
'''
with self.client.connect(*self.bestip):
market = get_stock_market(symbol)
result = self.client.get_finance_info(market=market, code=symbol)
return to_data(result)
def k(self, symbol='', begin=None, end=None):
'''
读取k线信息
:param symbol:
:param begin: 开始日期
:param end: 截止日期
:return: pd.dataFrame or None
'''
with self.client.connect(*self.bestip):
result = self.client.get_k_data(symbol, begin, end)
return to_data(result)
def index(self,
symbol='000001',
market=MARKET_SH,
frequency='9',
start=1,
offset=2):
'''
获取指数k线
K线种类:
- 0 5分钟K线
- 1 15分钟K线
- 2 30分钟K线
- 3 1小时K线
- 4 日K线
- 5 周K线
- 6 月K线
- 7 1分钟
- 8 1分钟K线
- 9 日K线
- 10 季K线
- 11 年K线
:param symbol: 股票代码
:param frequency: 数据类别
:param market: 证券市场
:param start: 开始位置
:param offset: 每次获取条数
:return: pd.dataFrame or None
'''
with self.client.connect(*self.bestip):
result = self.client.get_index_bars(int(frequency), int(market),
str(symbol), int(start),
int(offset))
return to_data(result)
def block(self, tofile="block.dat"):
'''
获取证券板块信息
:param tofile:
:return: pd.dataFrame or None
'''
with self.client.connect(*self.bestip):
result = self.client.get_and_parse_block_info(tofile)
return to_data(result)
class PytdxApi:
"""TDX数据服务类"""
def __init__(self):
"""Constructor"""
self.connect_status: bool = False
self.login_status: bool = False
self.hq_api = None # 行情API
self.conc_code_num = 50 # 并发获取行情的股票个数
# 行情订阅
self.active = False
self.run_subscribe = Thread(target=self.get_realtime_data)
self.symbols = list()
self.symbols_split = list()
def connect_api(self, host: str = "", port: int = 0):
"""连接行情api"""
# 连接行情API并检查连接情况
try:
if not self.connect_status:
self.hq_api = TdxHq_API()
self.hq_api.connect(host, port)
self.connect_status = True
self.login_status = True
self.subscribe_start()
except Exception as e:
return e
def get_realtime_quotes(self, quotes_list: list):
"""获取实时行情数据"""
data = self.hq_api.get_security_quotes(quotes_list)
return data
def get_realtime_data(self):
"""获取实时行情切片"""
try:
while self.active:
if not self.symbols_split:
sleep(1)
continue
data = list()
for symbols in self.symbols_split:
d = self.get_realtime_quotes(symbols)
data.extend(d)
self.on_tick_data(data)
sleep(2)
except:
error = dict()
error["error_id"] = "pytdx"
error["error_msg"] = "行情订阅失败"
self.on_error(error)
def get_transaction_count(self, market: int) -> int:
"""
查询市场标的数量
"""
return self.hq_api.get_security_count(market)
def get_transaction_list(self, market: int, start: int) -> list:
"""查询市场标的列表"""
return self.hq_api.get_security_list(market, start)
def subscribe_start(self):
"""启动行情订阅"""
self.active = True
self.run_subscribe.start()
def subscribe(self, symbol: Any):
"""订阅行情数据"""
if isinstance(symbol, tuple):
if symbol not in self.symbols:
self.symbols.append(symbol)
elif isinstance(symbol, list):
for s in symbol:
if s not in self.symbols:
self.symbols.append(s)
else:
error = dict()
error["error_id"] = "pytdx"
error["error_msg"] = f"订阅标的代码格式不正确{symbol}"
self.on_error(error)
return
symbol_split = self.get_code_split()
self.symbols_split = copy(symbol_split)
def subscribe_close(self):
"""关闭订阅"""
if self.active:
self.active = False
self.run_subscribe.join()
def get_transaction_info(self):
"""获取所有合约信息"""
for exchange in list(exchange_map.values()):
count = self.get_transaction_count(exchange)
for c in range(0, count, 1000):
symbols = self.get_transaction_list(exchange, c)
for symbol in symbols:
symbol["exchange"] = exchange
if symbol["code"][:2] in ["60", "30", "688", "00"]:
symbol["product"] = 3
else:
symbol["product"] = 2
self.on_contract_info(symbol, False)
self.on_contract_info({"exchange": exchange}, True)
def get_all_stock(self):
"""获取所有股票数据"""
stocks = list()
for exchange in list(exchange_map.values()):
count = self.get_transaction_count(exchange)
for s in range(0, count, 1000):
d = self.get_transaction_list(exchange, s)
stocks.extend(d)
l = len(stocks) - 1
for i, stock in enumerate(stocks):
if stock["code"][:1] in ["60", "30", "688", "00"]:
if i == l:
self.on_contract_info(stock, True)
else:
self.on_contract_info(stock, False)
def on_contract_info(self, data: dict, last: bool) -> None:
""""""
pass
def on_tick_data(self, data):
"""订阅数据处理"""
pass
def on_error(self, err):
"""接口错误处理"""
pass
@staticmethod
def generate_symbols(symbols: list):
"""组装symbols数据，pytdx接收的是以市场代码和标的代码组成的元祖的list"""
new_symbols = []
for symbol in symbols:
code, exchange = symbol.split(".")
new_symbol = (exchange_map[exchange], code)
new_symbols.append(new_symbol)
return new_symbols
@staticmethod
def get_fast_ip():
"""获取最快IP"""
host = "210.51.39.201"
port = 7709
return host, port
@staticmethod
def check_symbol(symbol: str):
"""检查标的格式"""
if symbol:
code, market = symbol.split(".")
market = exchange_map.get(market)
return code, market
else:
return False
def get_code_split(self):
"""获得切割好的股票代码段"""
code_split_list = []
for i in range(0, len(self.symbols) + 1, self.conc_code_num):
code_split = self.symbols[i : i + self.conc_code_num]
code_split_list.append(code_split)
return code_split_list
def exit(self):
"""数据服务关闭"""
# 关闭订阅
self.subscribe_close()
# 关闭接口
self.login_status = False
self.connect_status = False
self.hq_api.disconnect()
self.hq_api = None
data2 = api.get_xdxr_info(1, '600300')
print(data2)
print("获取股票行情")
stocks = api.get_security_quotes([(0, "000002"), (1, "600300")])
print(stocks)
print("获取k线")
data = api.get_security_bars(9, 0, '000001', 4, 3)
print(data)
print("获取 深市 股票数量")
print(api.get_security_count(0))
print("获取股票列表")
stocks = api.get_security_list(1, 255)
print(stocks)
print("获取指数k线")
data = api.get_index_bars(9, 1, '000001', 1, 2)
print(data)
print("查询分时行情")
data = api.get_minute_time_data(1, '600300')
print(data)
print("查询历史分时行情")
data = api.get_history_minute_time_data(1, '600300', 20161209)
print(data)
print("查询分时成交")
data = api.get_transaction_data(1, '000002', 0, 30)
print(data)
print("查询历史分时成交")
data = api.get_history_transaction_data(2, '600302', 0, 10, 20170209)
class TdxData:
def __init__(self):
self.api = TdxHq_API(heartbeat=True)
self.dbUtil = DBUtil.getInstance()
def get_security_quotes(self, code, type):
return self.api.get_security_quotes([(type, code)])
# 支持板块及个股
def days(self, code, type, bk=False, all=False, day=5):
category = int(config.getByKey('TDX_CATEGORY'))
try:
with self.api.connect(TDX_IP, TDX_PORT):
data = []
if all:
if bk:
for i in range(10):
data += self.api.get_index_bars(
category, type, code, (9 - i) * 800, 800)
else:
for i in range(10):
data += self.api.get_security_bars(
category, type, code, (9 - i) * 800, 800)
if len(data) > 0:
df = self.api.to_df(data).drop([
'amount', 'year', 'month', 'day', 'hour', 'minute'
],
axis=1)
df['trade_date'] = df['datetime'].apply(
lambda x: x[0:10].replace('-', ''))
df = df.drop(['datetime'], axis=1)
df = df.sort_values(by=['trade_date'],
axis=0,
ascending=False)
return df
else:
return self.api.to_df(data)
else:
if bk:
data = self.api.get_index_bars(category, type, code, 0,
day) # 返回DataFrame
else:
data = self.api.get_security_bars(
category, type, code, 0, day)
if len(data) > 0:
df = self.api.to_df(data).drop([
'amount', 'year', 'month', 'day', 'hour', 'minute'
],
axis=1)
df['trade_date'] = df['datetime'].apply(
lambda x: x[0:10].replace('-', ''))
df = df.drop(['datetime'], axis=1)
df = df.sort_values(by=['trade_date'],
axis=0,
ascending=False)
return df
else:
return self.api.to_df(data)
except Exception as e:
logging.info("暂不支持类型,代码：%s:%s" % (code, e))
return self.api.to_df([])
# F10 查询公司信息目录
def get_company_info_category(self, code, type):
with self.api.connect(TDX_IP, TDX_PORT):
df = pd.DataFrame(self.api.get_company_info_category(type, code))
df['txt'] = None
return df
return []
def get_company_info_content(self, code, type, df):
with self.api.connect(TDX_IP, TDX_PORT):
return self.api.get_company_info_content(type, code,
df['filename'].values[0],
df['start'].values[0],
df['length'].values[0])
return ""
# 查询财务数据
def get_finance_info(self, code, type):
with self.api.connect(TDX_IP, TDX_PORT):
return self.api.get_finance_info(type, code)
return ''
# 每年更新一次，板块个股关系
def updateBk(self):
with self.api.connect(TDX_IP, TDX_PORT):
"""
# 获取股票所属板块信息
# 板块相关参数
BLOCK_SZ = "block_zs.dat"
BLOCK_FG = "block_fg.dat"
BLOCK_GN = "block_gn.dat"
BLOCK_DEFAULT = "block.dat"
"""
bk_zs = self.api.to_df(
self.api.get_and_parse_block_info("block_zs.dat")) #指数板块
bk_fg = self.api.to_df(
self.api.get_and_parse_block_info("block_fg.dat")) #风格板块
bk_gn = self.api.to_df(
self.api.get_and_parse_block_info("block_gn.dat")) #概念板块
bk_default = self.api.to_df(
self.api.get_and_parse_block_info("block.dat")) # 默认
self.dbUtil.to_sql(bk_gn,
'stock_bk_gn',
if_exists='replace',
index=False,
dtype=STOCK_BK_DTYPE)
self.dbUtil.to_sql(bk_fg,
'stock_bk_fg',
if_exists='replace',
index=False,
dtype=STOCK_BK_DTYPE)
self.dbUtil.to_sql(bk_zs,
'stock_bk_zs',
if_exists='replace',
index=False,
dtype=STOCK_BK_DTYPE)
self.dbUtil.to_sql(bk_default,
'stock_bk_default',
if_exists='replace',
index=False,
dtype=STOCK_BK_DTYPE)
# 获取股票列表
tmp1 = self.api.to_df(self.api.get_security_list(0, 0)) # 深圳
tmp1['type'] = 0
tmp2 = self.api.to_df(self.api.get_security_list(1, 0)) # 上海
tmp2['type'] = 1
tmp = tmp1.append(tmp2)
self.dbUtil.to_sql(tmp,
'stock_bk',
if_exists='replace',
index=False,
dtype=STOCK_BK)
def updateGD(self, code, type):
url = 'http://emweb.securities.eastmoney.com/PC_HSF10/ShareholderResearch/ShareholderResearchAjax?code=%s%s' % (
type.lower(), code)
html = urllib.request.urlopen(url).read()
# 将字符串转换成字典
data = json.loads(html.decode('utf-8'))
# gdrs 股东人数,sdgd 十大股东 ，sdltgd 十大流通股东
df_gdrs = pd.DataFrame(data['gdrs'])
df_gdrs['code'] = code
try:
db_df_gdrs = self.dbUtil.read_sql(
"select * from stock_gdrs where code ='%s'" % code)
# 数据合并
df_gdrs = df_gdrs.append(db_df_gdrs).drop_duplicates(
subset=['code', 'rq', 'gdmc'], keep='last')
except Exception as e:
pass
self.dbUtil.to_sql(df_gdrs,
'stock_gdrs',
if_exists='append',
index=False,
dtype=STOCK_GDRS_DTYPE)
sdgd = []
for i in range(len(data['sdgd'])):
sdgd += data['sdgd'][i]['sdgd']
df_sdgd = pd.DataFrame(sdgd)
df_sdgd['code'] = code
try:
db_df_sdgd = self.dbUtil.read_sql(
"select * from stock_sdgd where code ='%s'" % code)
df_sdgd = df_sdgd.append(db_df_sdgd).drop_duplicates(
subset=['code', 'rq', 'gdmc'], keep='last')
except Exception as e:
pass
self.dbUtil.to_sql(df_sdgd,
'stock_sdgd',
if_exists='append',
index=False,
dtype=STOCK_SDGD_DTYPE)
sdltgd = []
for i in range(len(data['sdltgd'])):
sdltgd += data['sdltgd'][i]['sdltgd']
df_sdltgd = pd.DataFrame(sdltgd)
df_sdltgd['code'] = code
# 获取后与数据库中的数据进行merge,首次表不存在，会抛异常
try:
db_df_sdltgd = self.dbUtil.read_sql(
"select * from stock_sdltgd where code ='%s'" % code)
df_sdltgd = df_sdltgd.append(db_df_sdltgd).drop_duplicates(
subset=['code', 'rq', 'gdmc'], keep='last')
except Exception as e:
pass
self.dbUtil.to_sql(df_sdltgd,
'stock_sdltgd',
if_exists='append',
index=False,
dtype=STOCK_SDGD_DTYPE)
# 没季度更新一次
def updateGDs(self):
codes = self.dbUtil.read_sql("select ts_code from stock_basic")
tmp = codes['ts_code'].str.split('.', expand=True)
for index, row in tmp.iterrows():
try:
self.updateGD(row[0], row[1])
logging.info('%s更新结束,当前索引%s' % (row[0], index))
except Exception as e:
logging.info('%s更新失败,当前索引%s' % (row[0], index))
# 分红
# 分红地址http://data.eastmoney.com/yjfp/201812.html
def updateFh(self, rq):
url = 'http://data.eastmoney.com/DataCenter_V3/yjfp/getlist.ashx?filter=(ReportingPeriod=^%s^)' % rq
html = requests.get(url)
# 将字符串转换成字典
data = json.loads(html.text)['data']
if len(data) == 0:
return 0
df = pd.DataFrame(data)
df['ReportingPeriod'] = df['ReportingPeriod'].apply(lambda x: x[0:10])
# 首次需要将df_fh制空，因为表还不存在
if self.dbUtil.is_exist("stock_fh"):
db_fh = self.dbUtil.read_sql(
"select * from stock_fh where ReportingPeriod = '%s'" %
df['ReportingPeriod'][0])
if db_fh.empty: # 不存在当前日期的分红信息，进行拼接
self.dbUtil.to_sql(df,
'stock_fh',
if_exists='append',
index=False)
return 1
else:
pass
else:
self.dbUtil.to_sql(df, 'stock_fh', if_exists='append', index=False)
return 1
# 更新历年分红
def updateFhYears(self):
now = int(time.strftime("%Y", time.localtime())) + 1
lastYear = int(self._getFhMaxYear())
for i in range(lastYear, now): #初始化时开启
type = self.updateFh('%s-06-30' % i)
logging.info('%s-06-30%s' % (i, '成功' if type == 1 else '失败'))
self.updateFh('%s-12-31' % i)
logging.info('%s-12-31%s' % (i, '成功' if type == 1 else '失败'))
def _getFhMaxYear(self):
if self.dbUtil.is_exist('stock_fh'):
try:
df = self.dbUtil.read_sql(
'select substr(max(t.ReportingPeriod ),0,5) year from stock_fh t'
)
return df['year'][0].values
except Exception as e:
pass
return 1991
@classmethod
def getInstance(cls):
if not hasattr(TdxData, "_instance"):
TdxData._instance = TdxData()
return TdxData._instance
class TDX(object):
'''
This class is tong da xin data source.
We can use it to get down the stock datas.
Tushare can't get minter line and or year line.
TDX can search index of stock and funds.
'''
def __init__(self):
self.tdx_api = TdxHq_API()
self.__ip = '119.147.212.81' #输入IP
self.__port = 7709 #端口
self.__code = '600200'
self.__market = 1 #市场代码 0:深圳，1:上海
self._startdate = "2017-01-01"
self.today = datetime.date.today()
self._enddate = datetime.datetime.strftime(self.today, '%Y-%m-%d')
self.__mkt_segment = {
'sh': '60',
"sz": '00',
"cyb": "30",
} #segment 当前板块开始字符串
def __str__(self):
return 'TDX object (code : %s)' % self.code
@property
def IP(self): # self.IP
return self.__ip
@property
def PORT(self):
return self.__port
@property
def code(self): #定义stock code 属性
return self.__code
@code.setter #设定code
def code(self, code_input):
"""
The setter of the code property
"""
if not isinstance(code_input, str): #确定是否是字符串
raise ValueError('the code must string!')
if not len(code_input) == 6: #确定长度
raise ValueError('the code value error,the len must SIX !')
if code_input.startswith('60'): #确定表头
self.__market = 1
elif code_input.startswith('00'):
self.__market = 0
elif code_input.startswith('30'):
self.__market = 0
else:
raise ValueError('this code is not stock code')
self.__code = code_input
@property
def startdate(self): #开始日期
return self._startdate
@startdate.setter #设置日期
def startdate(self, date_input):
"""
The setter of the start date property
"""
if not isinstance(date_input, str):
raise ValueError('the date must string!')
if not len(date_input) == 8:
raise ValueError(
'the date value error,the date formet must xxxx-xx-xx !')
self._startdate = date_input
@property #结束日期
def enddate(self):
return self._enddate
@enddate.setter
def enddate(self, date_input):
"""
The setter of the start date property
"""
if not isinstance(date_input, str):
raise ValueError('the date must string!')
if not len(date_input) == 8:
raise ValueError(
'the date value error,the date formet must xxxx-xx-xx !')
self._enddate = date_input
def get_day_data_tdx(self): #获取K line
with self.tdx_api.connect(self.IP, self.PORT):
data = self.tdx_api.get_k_data(self.code, self.startdate,
self.enddate)
data = pandas.DataFrame(data)
data.date = data.date.apply(
lambda x: datetime.datetime.strptime(x, "%Y-%m-%d"))
return data
#TODO: 现在是用800点进行计数，以后会细化功能
def get_k_data_tdx(self, k_mode=9):
"""
获取k 线图，总计800 点
Parameters
----------
k_mode= 0-11
0 5分钟K线
1 15分钟K线
2 30分钟K线
3 1小时K线
4 日K线
5 周K线
6 月K线
7 1分钟
8 1分钟K线 9 日K线
10 季K线
11 年K线
Returns
-------
"""
with self.tdx_api.connect(self.self.IP, self.self.PORT):
data = self.tdx_api.get_security_bars(k_mode, self.__market,
self.code, 0, 800)
data = pandas.DataFrame(data)
#data.date = data.date.apply(
# lambda x: datetime.datetime.strptime(x, "%Y-%m-%d"))
return data
def len_market(self): #市场有多少只股票
with self.tdx_api.connect(self.IP, self.PORT):
_len = self.tdx_api.get_security_count(self.__market)
return _len
def get_page_tdx(self, block=None):
if block is None:
market = 1
page = [0]
elif block in ['sh', 'SH']:
market = 1
page = [13, 14]
elif block in ['sz', 'SZ']:
print('block for shenzhen')
market = 0
page = [0, 1]
elif block in ['cyb', 'CYB']:
print('block for chuang ye ban')
market = 0
page = [7, 8]
else:
pass
code_list_df = pandas.DataFrame()
with self.tdx_api.connect(self.IP, self.PORT):
for pn in page:
data = self.tdx_api.get_security_list(market, pn * 1000)
data = pandas.DataFrame(data)
print(data)
code_list_df = code_list_df.append(data, ignore_index=True)
return code_list_df
def get_base_finace_tdx(self):
with self.tdx_api.connect(self.IP, self.PORT):
data = self.tdx_api.get_finance_info(0, '000001')
data = pandas.Series(data)
print(data)
def get_min_data(self):
from pytdx.params import TDXParams
with self.tdx_api.connect(self.IP, self.PORT):
data = self.tdx_api.get_history_minute_time_data(
TDXParams.MARKET_SH, self.code, 20161209)
data = pandas.DataFrame(data)
print(data)
#TODO: 需要确定 0: buy 1 : sell
def get_tick_data(self):
"""
历史分笔交易：time 顺序； price ; vol ;buyorsell [1:] [0:];
sh 60 13000-14000
Parameters
----------
Returns
-------
"""
data = pandas.DataFrame()
with self.tdx_api.connect(self.IP, self.PORT):
for i in [2000, 0000]:
df = self.tdx_api.get_history_transaction_data(
TDXParams.MARKET_SH, "600547", i, 2000, 20160308)
df = pandas.DataFrame(df)
data = data.append(df, ignore_index=True)
return data
def get_tick_today(self):
"""
Get every time the each deal for today.每组数最大len 2 k 所以要确定的数据长度
Parameters
----------
self:
Returns
-------
"""
with self.tdx_api.connect(self.IP, self.PORT):
data = pandas.DataFrame()
for i in [0, 2000]:
df = self.tdx_api.get_transaction_data(self.__market,
self.code, i, 2000)
df = pandas.DataFrame(df)
data = data.append(df, ignore_index=True)
return data
def get_block(self):
with self.tdx_api.connect(self.IP, self.PORT):
data = self.tdx_api.get_and_parse_block_info("block.dat")
data = pandas.DataFrame(data)
print(data)
def get_market_segment_list(self, mkt):
data = self.get_page_tdx(mkt)
self.code_list = pandas.DataFrame()
pbar = tqdm(total=len(data.code))
mkt_hard = self.mkt_segment[mkt]
for idx, __code in enumerate(data.code):
pbar.update(1)
if __code.startswith(mkt_hard, 0, 2):
self.code_list = self.code_list.append(data.loc[idx],
ignore_index=True)
return self.code_list
def get_sh_list(self):
return self.get_market_segment_list('sh')
def get_sz_list(self):
return self.get_market_segment_list('sz')
def get_cyb_list(self):
return self.get_market_segment_list('cyb')
ttsengine = pyttsx3.init()
mylog.add("stock_trading_{time}.log", encoding='utf-8')
from pytdx.util.best_ip import select_best_ip
stock_ip = select_best_ip('stock')
print(stock_ip)
future_ip = select_best_ip('future')
print(future_ip)
# if api.connect('119.147.212.81', 7709):
if api.connect(stock_ip['ip'], stock_ip['port']):
szall = api.get_security_count(0)
shall = api.get_security_count(1)
szsecs = api.get_security_list(0, 0)
shsecs = api.get_security_list(1, 0)
# szStockProcess(DAY)
# shStockProcess(DAY)
print('全市场轮询结束.')
# data = api.get_k_data(stock, '2015-01-01', '2020-01-14')
# shsecdict = {}
# shall = api.get_security_count(1)
# for step in range(0, shall, 1000):
# shsecs = api.get_security_list(1, step)
# for sec in shsecs:
# if sec['code'].startswith('68') or sec['code'].startswith('60'):
# stock = sec['code']
# stockname = sec['name']
