def __exhq_ping(self, ip, port):
api = TdxExHq_API()
with api.connect(ip, port, time_out=0.7):
assert api.get_instrument_count() > 20000
api.disconnect()
return True
return False
def get_hk_quote_by_tdx(code_list):
"""
获取香港主板市场行情数据
需要使用拓展行情接口
通达信接口的港股行情数据比较简陋，只有当前价格，缺乏bid，ask以及对应的量能
:param code_list:
:return:
"""
api_ex = TdxExHq_API()
result = []
if api_ex.connect('140.143.179.226', 7727):
for code in code_list:
data = api_ex.get_instrument_quote(31, code)
result.append(
[data[0]["code"], data[0]["price"], data[0]["xianliang"]])
api_ex.disconnect()
df = pd.DataFrame(data=result, columns=["code", "price", "xianliang"])
return df
def __exhq_bars(self, code, offset, frequency=9):
assert self.__ex.qsize() > 0
api = TdxExHq_API()
ip, port = self.__ex.get()
with api.connect(ip, port):
df = list()
for _code in code:
for _start, _count in offset:
df.append(
api.to_df(
api.get_instrument_bars(
frequency,
self.__exhq_list().xs(_code).market, _code,
_start, _count)).assign(code=_code))
api.disconnect()
self.__ex.put((ip, port))
if len(df) < 1:
return None
return pandas.concat(df, sort=False)
class ExEngine:
def __init__(self, *args, **kwargs):
self.api = TdxExHq_API(args, kwargs)
def connect(self):
self.api.connect('61.152.107.141', 7727)
return self
def __enter__(self):
return self
def exit(self):
self.api.disconnect()
def __exit__(self, exc_type, exc_val, exc_tb):
self.api.disconnect()
@lazyval
def markets(self):
return self.api.to_df(self.api.get_markets())
def update_futures(args):
"""Update Future 1min data in MongoDB
Args:
Returns:
"""
client = pymongo.MongoClient(args.mongo_uri, serverSelectionTimeoutMS=1000)
client.server_info()
db = client[args.database]
collection = db['future_china_1min']
api = TdxExHq_API(heartbeat=True, multithread=True)
api.connect('61.152.107.141', 7727)
num = api.get_instrument_count()
insts = [api.get_instrument_info(i, QSIZE) for i in range(0, num, QSIZE)]
insts = [x for i in insts for x in i]
exchs = ['中金所期货', '上海期货', '大连商品', '郑州商品']
markets = [t['market'] for t in api.get_markets() if t['name'] in exchs]
futures = [t for t in insts
if t['market'] in markets and t['code'][-2] != 'L']
for future in futures:
qeury = collection.find({"ticker": future['code']})
qeury = qeury.sort('datetime', pymongo.DESCENDING)
qeury = qeury.limit(1)
last_one = list(qeury)
if len(last_one) > 0:
last_date = last_one[0]['datetime'] + timedelta(minutes=1)
else:
last_date = datetime.now() - timedelta(days=365)
end_date = datetime.now().date()
end_date = datetime.combine(end_date - timedelta(days=1),
time(ALL_MARKET_END_HOUR, 0))
_start_date = end_date
_bars = []
_pos = 0
while _start_date > last_date:
_res = api.get_instrument_bars(
TDXParams.KLINE_TYPE_1MIN,
future['market'],
future['code'],
_pos,
QSIZE)
try:
_bars = _res + _bars
except TypeError:
continue
_pos += QSIZE
if len(_res) > 0:
_start_date = _res[0]['datetime']
_start_date = datetime.strptime(_start_date, '%Y-%m-%d %H:%M')
else:
break
if len(_bars) == 0:
continue
data = api.to_df(_bars)
data = data.assign(datetime=pd.to_datetime(data['datetime']))
data = data.assign(ticker=future['code'])
data = data.drop(
['year', 'month', 'day', 'hour', 'minute', 'price', 'amount'],
errors='ignore',
axis=1)
data = data.rename(
index=str,
columns={
'position': 'oi',
'trade': 'volume',
})
data['date'] = pd.to_datetime(data['datetime'].dt.date)
_miss_ts = data['datetime'].dt.hour > ALL_MARKET_END_HOUR
data.loc[_miss_ts, 'datetime'] -= timedelta(days=1)
data = data.set_index('datetime', drop=False)
data = data[str(last_date):str(end_date)]
collection.insert_many(data.to_dict('records')) if len(data) > 0 else 0
_logger.info(future['code'])
api.disconnect()
class TdxMarket(MarketBase):
__market_name__ = 'TDX'
__timeframe__ = '1MIN'
def __init__(self, host: Optional[str] = None):
self._env = env = Env()
env.read_env()
tdx_host = host if host else env.str('TDX_HOST')
self._api = TdxExHq_API(heartbeat=True, multithread=True)
self._ip, self._port = tdx_host.split(':')
self._server_tz = timezone('Asia/Shanghai')
self._pill = Event()
self._market_mapping = dict(
zip(["SHFE", "CZCE", "DCE", "CFFEX", "US"], [30, 28, 29, 47, 74]))
def connect(self):
if not self._api.connect(self._ip, int(self._port)):
raise RemoteError('%s:%s connect error.' %
(self.ip, int(self.port)))
self._market_list = list(
self._api.to_df(self._api.get_markets()).market)
async def watch_klines(self, symbol: str):
market, code = symbol.split('.')
last_kline = None
while not self._pill.is_set():
tdx_klines = self._get_instrument_bars(
TDXParams.KLINE_TYPE_1MIN, self._market_mapping[market], code,
0, 10)
# 这个地方处理的原因在于由于是轮询的，就有可能出现在轮询间隔的时间(这里是1秒)内，有一半时间属于上一根K线，有一半时间属于下一根K线，所以如果我们只看最新的K线的话，前一根K线的一部分数据可能就会丢失，因此这里判断一下前一根K线如果有更新，就先把前一根K线推送了，再推送下一根K线
prev_kline = self._parse_kline(tdx_klines[-2])
latest_kline = self._parse_kline(tdx_klines[-1])
if last_kline and prev_kline['datetime'] == last_kline['datetime']:
self.logger.debug('yield prev kline: %s' % prev_kline)
yield prev_kline
if last_kline != latest_kline:
yield latest_kline
self.logger.debug('yield latest kline: %s' % latest_kline)
last_kline = latest_kline
await sleep(1)
async def get_kline_histories(self,
symbol: str,
from_ts: Optional[int] = None,
limit: Optional[int] = None):
market, code = symbol.split('.')
if self._market_mapping[market] in self._market_list:
bars = []
if from_ts is not None:
idx = 0
while len(bars) == 0 or bars[-1]['datetime'] >= from_ts:
bars += [
self._parse_kline(bar) for bar in reversed(
self._get_instrument_bars(
TDXParams.KLINE_TYPE_1MIN,
self._market_mapping[market], code, idx *
700, 700))
]
idx += 1
# 前面都是整700地取，所以有可能取到的数据会超过 from_ts,因此这里再做一次过滤
bars = list(filter(lambda x: x['datetime'] >= from_ts, bars))
elif limit is not None:
for i in range(limit // 700):
bars += [
self._parse_kline(bar) for bar in reversed(
self._get_instrument_bars(
TDXParams.KLINE_TYPE_1MIN,
self._market_mapping[market], code, i *
700, 700))
]
bars += [
self._parse_kline(bar) for bar in reversed(
self._get_instrument_bars(TDXParams.KLINE_TYPE_1MIN,
self._market_mapping[market],
code, limit // 700 *
700, limit % 700))
]
return reversed(bars)
def disconnect(self):
self._pill.set()
self._api.disconnect()
def _get_instrument_bars(self, category, market, code, start=0, count=700):
while True:
try:
cmd = GetInstrumentBars(self._api.client, lock=self._api.lock)
cmd.setParams(category, market, code, start=start, count=count)
return cmd.call_api()
except Exception as e:
pass
def _parse_kline(self, kline_tdx: dict) -> dict:
dt = self._server_tz.localize(
datetime.strptime(kline_tdx['datetime'],
'%Y-%m-%d %H:%M')).astimezone(pytz.utc)
return {
'datetime': int(dt.timestamp() * 1000),
'open': str(round(kline_tdx['open'], 2)),
'high': str(round(kline_tdx['high'], 2)),
'low': str(round(kline_tdx['low'], 2)),
'close': str(round(kline_tdx['close'], 2)),
'volume': kline_tdx['trade'],
}
class data(object):
def __init__(self, heartbeat=True, isstock=False):
self.api = None
self.isstock = isstock
self.heartbeat = heartbeat
self.number = 80000
self.market = None
self.datatype = 0
self.result = []
self.TDX_IP_SETS = [
"119.147.86.168",
'119.97.185.5'
'202.103.36.71',
'139,196,185,253',
'61.152.107.171',
]
# 218.75.74.103:7721 60.12.15.21:7721
self.TDX_IP_SETS_STOCK = [
'119.147.164.60', '218.75.126.9', '115.238.90.165',
'124.160.88.183', '60.12.136.250', '218.108.98.244',
'218.108.47.69', '14.17.75.71', '180.153.39.51'
]
self.file_incon = FILE_INCON
self.file_tdxhy = FILE_TDXHY
self.file_tdxzs = FILE_TDXZS
self.weight = {}
def _get_incon(self, ):
'''获取行业分类代码
'''
f = open(self.file_incon, "rb")
data = f.read()
strings = data.decode("gbk",
'ignore').rstrip("\x00").replace("\r\n", "\n")
data = strings.split("######")
rst = {}
for hystr in data:
key = re.findall(r'#.*', hystr)
if key == ['#TDXNHY']:
hylst = hystr.replace("#TDXNHY", "").strip("\n").split("\n")
for item in hylst:
k, v = item.split("|")
rst[k] = [v]
return rst
def _get_tdxhy(self, islocal=True):
'''获取股票和行业对应列表
'''
if islocal:
stocklist = HY_WEIGHT.keys()
else:
stocklist = list(ts.get_stock_basics().index) #获取全市场股票代码
rst = self._get_incon()
f = open(self.file_tdxhy, "rb")
data = f.read().decode("gbk", 'ignore').rstrip("\x00").replace(
"\r\n", "\n").strip("\n").split("\n")
for i in data:
_, code, tdxhy, _, _ = i.split("|")
if tdxhy != "T00" and code in stocklist:
rst[tdxhy].append(code)
return rst
def _get_tdxzs(self, islocal=True):
'''生成通达性版块代码对应股票列表
'''
dct = {}
rst = self._get_tdxhy(islocal=islocal)
f = open(self.file_tdxzs, "rb")
data = f.read().decode("gbk", 'ignore').rstrip("\x00").replace(
"\r\n", "\n").strip("\n").split("\n")
for i in data:
name, code, _, _, _, hy = i.split("|")
code = int(code)
if 880301 <= code and 880497 >= code and hy in rst.keys():
k = hy[:5]
if not dct.__contains__(k):
dct[k] = {"name": "", "code": "", "stocklist": []}
if k == hy:
dct[k]["name"] = name
dct[k]["code"] = code
dct[k]["stocklist"].extend(rst[hy][1:])
return dct
def get_tdxhy_list(self, islocal=True):
'''获取通达信行业板块指数对应的股票列表
'''
return self._get_tdxzs(islocal)
def get_weight(self, htlist={}, islocal=True):
'''获取行业板块个股权重，流动市值为权重系数
备注：回测是为方便处理，以最后一天的权重系数作为历史上的权重
'''
if islocal:
self.weight = HY_WEIGHT
else:
if not htlist:
htlist = self.get_tdxhy_list(islocal)
tasks = []
for v in htlist.values():
tasks.append(gevent.spawn(self.get_latest_ltsz,
v["stocklist"]))
gevent.joinall(tasks)
return self.weight
def get_latest_ltsz(self, stocks=[]):
'''获取最新流通市值,千万为单位，取整
'''
unit = 10000000
for code in stocks:
mk = self._select_market_code(code)
print(mk, code)
try:
ltgb = self.api.get_finance_info(mk, code)["liutongguben"]
price = self.api.get_security_bars(4, mk, code, 0,
1)[0]["close"]
ltsz = int(ltgb * price / unit)
self.weight[code] = ltsz
except:
print("*****", code)
return self.weight
@property
def mdb(self):
'''设置数据库连接
'''
if not hasattr(self, "_db"):
self._db = MongoDB()
return self._db
@property
def starttime(self):
'''采样起始时间
'''
if not hasattr(self, "_starttime"):
self._starttime = "2006-01-01"
return self._starttime
@starttime.setter
def starttime(self, value):
self._starttime = value
return self._starttime
@property
def endtime(self):
'''采样结束时间
'''
if not hasattr(self, "_endtime"):
self._endtime = str(datetime.date.today())
return self._endtime
@endtime.setter
def endtime(self, value):
self._endtime = value
return self._endtime
def getdata_m(
self,
collection,
db=MIN_STOCK_DB,
project={
"open": 1,
"high": 1,
"low": 1,
"close": 1,
"datetime": 1,
"vol": 1,
"_id": 0
}):
'''从数据库获取分钟数据
'''
filt = {"datetime": {"$gte": self.starttime, "$lte": self.endtime}}
data = [
i for i in self.mdb._dbclient(db)[collection].find(filt, project)
]
return pd.DataFrame(data)
def getcollections(self, db=MIN_STOCK_DB):
data = self.mdb.getallcollections(db)
return data
def clean_m(self, df, field="datetime"):
'''数据库数据有些格式有问题，比如，有些11.30有数据，有些没有。有些13.00有数据，有些没有
导致用分钟数据生成30分钟数据时出现异常
'''
df[field] = df[field].str.replace("13:00", "11:30")
df.set_index(field, inplace=True, drop=False)
df.index = pd.DatetimeIndex(df.index)
df.sort_index(inplace=True)
df["time"] = pd.date_range('1/1/2011 00-01-00',
periods=df.shape[0],
freq='T')
return df
def createdir(self, path):
'''创建输出目录
'''
if not os.path.exists(path):
os.makedirs(path)
return
def setmarket(self):
'''设置商品市场代码
'''
market = []
for i in range(100):
market += self.api.get_instrument_info(i * 500, 500)
self.market = self.api.to_df(market)
return self.market
def connect(self):
if self.isstock:
self.api = TdxHq_API(heartbeat=self.heartbeat)
port = 7709
TDX_IP_SETS = self.TDX_IP_SETS_STOCK
else:
self.api = TdxExHq_API(heartbeat=self.heartbeat)
port = 7727
TDX_IP_SETS = self.TDX_IP_SETS
for ip in TDX_IP_SETS:
try:
if self.api.connect(ip, port):
return
except:
pass
def disconnect(self):
self.api.disconnect()
def getdata(self, product="ICL8", market=47, number=80000, pn=400):
if product[0] in ["0", "3", "6"]:
info = self.fetch_get_stock_xdxr(product)
data = self.getdata_stock(product, number=number, pn=pn)
data.drop(data[data["close"] <= 0].index)
df = self.qfq(data, info)
elif product[0] in [
"8",
]:
df = self.getdata_block_index(product, market, number=number)
else:
df = self.getdata_future(product, market, number=number)
return df
def getdata_block_index(self,
code="000001",
market=1,
number=30000,
pn=500):
data = []
start = False
for i in range(int(number / pn) + 1):
temp = self.api.get_index_bars(self.datatype, market, code,
(int(number / pn) - i) * pn, pn)
if temp and len(temp) > 0:
start = True
if start and (not temp or len(temp) < pn):
for _ in range(3):
temp = self.api.get_index_bars(self.datatype, market, code,
(int(number / pn) - i) * pn,
pn)
if len(temp) < pn:
print(111111111111, pn - len(temp))
else:
break
try:
data += temp
except:
self.connect()
df = self.api.to_df(data)[["open", "close", "high", "low", "datetime"]]
df.set_index("datetime", inplace=True, drop=False)
return df
def getdata_future(self, product="ICL8", market=47, number=80000, pn=400):
data = []
start = False
for i in range(int(number / pn) + 1):
temp = self.api.get_instrument_bars(self.datatype, market, product,
(int(number / pn) - i) * pn,
pn)
if temp and len(temp) > 0:
start = True
if start and (not temp or len(temp) < pn):
for _ in range(3):
temp = self.api.get_instrument_bars(
self.datatype, market, product,
(int(number / pn) - i) * pn, pn)
try:
if len(temp) < pn:
print(111111111111, pn - len(temp))
else:
break
except:
self.connect()
try:
data += temp
except:
self.connect()
df = self.api.to_df(data)[["open", "close", "high", "low", "datetime"]]
df.set_index("datetime", inplace=True, drop=False)
return df
def set_main_rate(self, df_m, product="RBL8", f="MainContract.csv"):
'''商品主月除权
'''
df = pd.read_csv(f, encoding="gb2312")
df_p = df[df["ContractCode"] == product[:-2]]
lstr = " 15:00"
df_p = df_p.assign(
datetime=df_p['EndDate'].apply(lambda x: str(x)[0:10] + lstr))
df_p.set_index("datetime", inplace=True)
df_p.fillna(0, inplace=True)
df_m.loc[:, "date"] = df_m["datetime"].apply(lambda x: str(x)[0:10])
filt = df_m.index.isin(df_p.index)
df_m.loc[filt, "OpenPrice"] = df_p["OpenPrice"]
df_m.loc[filt, "Term"] = df_p["Term"]
df_m["OpenPrice"].fillna(method="bfill", inplace=True)
df_m["Term"].fillna(method="bfill", inplace=True)
df_m.dropna(inplace=True)
filt = (df_m["OpenPrice"]!=df_m["open"])&\
(df_m["date"]>df_m["date"].shift(1))&\
(abs(1-df_m["open"]/df_m["OpenPrice"])>0.008)&\
(df_m["OpenPrice"]>0)
df_m.loc[filt, "change"] = 1
df_m.loc[:, "adj"] = 1
rst = df_m[df_m["change"] > 0]
rst = (rst["Term"] != rst["Term"].shift(1))
filt = df_m.index.isin(rst[rst > 0].index)
df_m.loc[filt, "adj"] = df_m["open"] / df_m["OpenPrice"]
df_m.loc[:, 'adj'] = df_m["adj"].shift(-1)
df_m.loc[:, 'adj'] = df_m["adj"][::-1].cumprod()
print(df_m[(df_m["change"] > 0) | (df_m["change"].shift(-1) > 0)][[
"close", "adj", "open", "OpenPrice", "Term"
]])
df_m.loc[:, 'open'] = df_m['open'] * df_m['adj']
df_m.loc[:, 'high'] = df_m['high'] * df_m['adj']
df_m.loc[:, 'low'] = df_m['low'] * df_m['adj']
df_m.loc[:, 'close'] = df_m['close'] * df_m['adj']
return df_m
def getdata_stock(self, code="000001", number=30000, pn=500):
market = self._select_market_code(code)
data = []
for i in range(int(number / pn) + 1):
data += self.api.get_security_bars(self.datatype, market, code,
(int(number / pn) - i) * pn, pn)
df = self.api.to_df(data)[["open", "close", "high", "low", "datetime"]]
df.set_index("datetime", inplace=True, drop=False)
return df
def _select_market_code(self, code):
code = str(code)
if code[0] in ['5', '6', '9'] or code[:3] in [
"009", "126", "110", "201", "202", "203", "204"
]:
return 1
return 0
def fetch_get_stock_xdxr(self, code):
'除权除息'
market_code = self._select_market_code(code)
category = {
'1': '除权除息',
'2': '送配股上市',
'3': '非流通股上市',
'4': '未知股本变动',
'5': '股本变化',
'6': '增发新股',
'7': '股份回购',
'8': '增发新股上市',
'9': '转配股上市',
'10': '可转债上市',
'11': '扩缩股',
'12': '非流通股缩股',
'13': '送认购权证',
'14': '送认沽权证'
}
data = self.api.to_df(self.api.get_xdxr_info(market_code, code))
if len(data) >= 1:
data = data\
.assign(date=pd.to_datetime(data[['year', 'month', 'day']]))\
.drop(['year', 'month', 'day'], axis=1)\
.assign(category_meaning=data['category'].apply(lambda x: category[str(x)]))\
.assign(code=str(code))\
.rename(index=str, columns={'panhouliutong': 'liquidity_after',
'panqianliutong': 'liquidity_before', 'houzongguben': 'shares_after',
'qianzongguben': 'shares_before'})\
.set_index('date', drop=False, inplace=False)
if self.datatype == 0:
lstr = " 09:35"
elif self.datatype == 1:
lstr = " 09:45"
elif self.datatype == 2:
lstr = " 10:00"
elif self.datatype == 3:
lstr = " 10:30"
elif self.datatype == 4:
lstr = " 15:00"
elif self.datatype == 7:
lstr = " 09:31"
return data.assign(
date=data['date'].apply(lambda x: str(x)[0:10] + lstr))
else:
return None
def qfq(self, data, xdxr_data):
'''data: 除权前数据
info：除权信息
'''
start = data.index[0]
if xdxr_data is not None:
info = xdxr_data[xdxr_data["category"] == 1]
info.set_index("date", inplace=True)
df = pd.concat(
[data, info[['fenhong', 'peigu', 'peigujia', 'songzhuangu']]],
axis=1).fillna(0)
df['preclose'] = (df['close'].shift(1) * 10 - df['fenhong'] +
df['peigu'] * df['peigujia']) / (
10 + df['peigu'] + df['songzhuangu'])
df['adj'] = (df['preclose'].shift(-1) /
df['close']).fillna(1)[::-1].cumprod()
df['open'] = df['open'] * df['adj']
df['high'] = df['high'] * df['adj']
df['low'] = df['low'] * df['adj']
df['close'] = df['close'] * df['adj']
df['preclose'] = df['preclose'] * df['adj']
else:
df['preclose'] = df['close'].shift(1)
df['adj'] = 1
return df[start:]
def macdhandle(self,
df,
p5=5,
p15=15,
p30=30,
p60=60,
p240=240,
pweek=1200,
macd_f=12,
macd_s=26,
macd_m=9):
df.loc[:, "number"] = range(df.shape[0])
for i in [p5, p15, p30, p60, p240, pweek]:
pres = str(i) + "_"
#计算各周期初始macd
df.loc[::i, pres + "fEMA_mark"] = df["close"][::i].ewm(
adjust=False, span=macd_f).mean()
df.loc[::i, pres + "sEMA_mark"] = df["close"][::i].ewm(
adjust=False, span=macd_s).mean()
df.loc[::i, pres +
"DIFF_mark"] = df[pres + "fEMA_mark"] - df[pres +
"sEMA_mark"]
df[pres + "fEMA_mark"].fillna(method="ffill", inplace=True)
df[pres + "sEMA_mark"].fillna(method="ffill", inplace=True)
df[pres + "DIFF_mark"].fillna(method="ffill", inplace=True)
df[pres + "fEMA"] = (df[pres + "fEMA_mark"] *
(macd_f - 1) + df["close"] * 2) / (macd_f + 1)
df[pres + "sEMA"] = (df[pres + "sEMA_mark"] *
(macd_s - 1) + df["close"] * 2) / (macd_s + 1)
df.loc[::i, pres + "fEMA"] = df[pres + "fEMA_mark"]
df.loc[::i, pres + "sEMA"] = df[pres + "sEMA_mark"]
df.loc[:, pres + "DIFF"] = df[pres + "fEMA"] - df[pres + "sEMA"]
df.loc[::i, pres + "DEA_mark"] = df[pres + "DIFF"][::i].ewm(
adjust=False, span=macd_m).mean()
df[pres + "DEA_mark"].fillna(method="ffill", inplace=True)
df[pres +
"DEA"] = (df[pres + "DEA_mark"] *
(macd_m - 1) + df[pres + "DIFF"] * 2) / (macd_m + 1)
df.loc[::i, pres + "DEA"] = df[pres + "DEA_mark"]
df.loc[:,
pres + "MACD"] = 2 * (df[pres + "DIFF"] - df[pres + "DEA"])
df.loc[:, "DIF"], df.loc[:, "DEA"], df.loc[:, "MACD"] = talib.MACD(
df.close.values)
df.dropna(inplace=True) #丢弃前面NA数据
return df
def getblockstock(self, block="沪深300"):
'''股票版块对应股票列表
'''
df = self.api.to_df(self.api.get_and_parse_block_info("block.dat"))
stocks = list(df[df["blockname"] == block]["code"])
return stocks
'''
tdx扩展行情接口API
https://rainx.gitbooks.io/pytdx/content/pytdx_exhq.html
'''
#首先需要引入
from pytdx.exhq import TdxExHq_API
#然后，创建对象
api = TdxExHq_API()
#连接行情服务器
api.connect('106.14.95.149', 7727, time_out=30)
#获取市场代码
df=api.to_df(api.get_markets())
#输出信息
print(df)
df2=api.get_history_minute_time_data(31, "00020", 20170811)
df3=api.to_df(df2)
print(df3)
#断开连接
api.disconnect()
