def __exhq_ping(self, ip, port): api = TdxExHq_API() with api.connect(ip, port, time_out=0.7): assert api.get_instrument_count() > 20000 api.disconnect() return True return False
def get_hk_quote_by_tdx(code_list): """ 获取香港主板市场行情数据 需要使用拓展行情接口 通达信接口的港股行情数据比较简陋,只有当前价格,缺乏bid,ask以及对应的量能 :param code_list: :return: """ api_ex = TdxExHq_API() result = [] if api_ex.connect('140.143.179.226', 7727): for code in code_list: data = api_ex.get_instrument_quote(31, code) result.append( [data[0]["code"], data[0]["price"], data[0]["xianliang"]]) api_ex.disconnect() df = pd.DataFrame(data=result, columns=["code", "price", "xianliang"]) return df
def __exhq_bars(self, code, offset, frequency=9): assert self.__ex.qsize() > 0 api = TdxExHq_API() ip, port = self.__ex.get() with api.connect(ip, port): df = list() for _code in code: for _start, _count in offset: df.append( api.to_df( api.get_instrument_bars( frequency, self.__exhq_list().xs(_code).market, _code, _start, _count)).assign(code=_code)) api.disconnect() self.__ex.put((ip, port)) if len(df) < 1: return None return pandas.concat(df, sort=False)
class ExEngine: def __init__(self, *args, **kwargs): self.api = TdxExHq_API(args, kwargs) def connect(self): self.api.connect('61.152.107.141', 7727) return self def __enter__(self): return self def exit(self): self.api.disconnect() def __exit__(self, exc_type, exc_val, exc_tb): self.api.disconnect() @lazyval def markets(self): return self.api.to_df(self.api.get_markets())
def update_futures(args): """Update Future 1min data in MongoDB Args: Returns: """ client = pymongo.MongoClient(args.mongo_uri, serverSelectionTimeoutMS=1000) client.server_info() db = client[args.database] collection = db['future_china_1min'] api = TdxExHq_API(heartbeat=True, multithread=True) api.connect('61.152.107.141', 7727) num = api.get_instrument_count() insts = [api.get_instrument_info(i, QSIZE) for i in range(0, num, QSIZE)] insts = [x for i in insts for x in i] exchs = ['中金所期货', '上海期货', '大连商品', '郑州商品'] markets = [t['market'] for t in api.get_markets() if t['name'] in exchs] futures = [t for t in insts if t['market'] in markets and t['code'][-2] != 'L'] for future in futures: qeury = collection.find({"ticker": future['code']}) qeury = qeury.sort('datetime', pymongo.DESCENDING) qeury = qeury.limit(1) last_one = list(qeury) if len(last_one) > 0: last_date = last_one[0]['datetime'] + timedelta(minutes=1) else: last_date = datetime.now() - timedelta(days=365) end_date = datetime.now().date() end_date = datetime.combine(end_date - timedelta(days=1), time(ALL_MARKET_END_HOUR, 0)) _start_date = end_date _bars = [] _pos = 0 while _start_date > last_date: _res = api.get_instrument_bars( TDXParams.KLINE_TYPE_1MIN, future['market'], future['code'], _pos, QSIZE) try: _bars = _res + _bars except TypeError: continue _pos += QSIZE if len(_res) > 0: _start_date = _res[0]['datetime'] _start_date = datetime.strptime(_start_date, '%Y-%m-%d %H:%M') else: break if len(_bars) == 0: continue data = api.to_df(_bars) data = data.assign(datetime=pd.to_datetime(data['datetime'])) data = data.assign(ticker=future['code']) data = data.drop( ['year', 'month', 'day', 'hour', 'minute', 'price', 'amount'], errors='ignore', axis=1) data = data.rename( index=str, columns={ 'position': 'oi', 'trade': 'volume', }) data['date'] = pd.to_datetime(data['datetime'].dt.date) _miss_ts = data['datetime'].dt.hour > ALL_MARKET_END_HOUR data.loc[_miss_ts, 'datetime'] -= timedelta(days=1) data = data.set_index('datetime', drop=False) data = data[str(last_date):str(end_date)] collection.insert_many(data.to_dict('records')) if len(data) > 0 else 0 _logger.info(future['code']) api.disconnect()
class TdxMarket(MarketBase): __market_name__ = 'TDX' __timeframe__ = '1MIN' def __init__(self, host: Optional[str] = None): self._env = env = Env() env.read_env() tdx_host = host if host else env.str('TDX_HOST') self._api = TdxExHq_API(heartbeat=True, multithread=True) self._ip, self._port = tdx_host.split(':') self._server_tz = timezone('Asia/Shanghai') self._pill = Event() self._market_mapping = dict( zip(["SHFE", "CZCE", "DCE", "CFFEX", "US"], [30, 28, 29, 47, 74])) def connect(self): if not self._api.connect(self._ip, int(self._port)): raise RemoteError('%s:%s connect error.' % (self.ip, int(self.port))) self._market_list = list( self._api.to_df(self._api.get_markets()).market) async def watch_klines(self, symbol: str): market, code = symbol.split('.') last_kline = None while not self._pill.is_set(): tdx_klines = self._get_instrument_bars( TDXParams.KLINE_TYPE_1MIN, self._market_mapping[market], code, 0, 10) # 这个地方处理的原因在于由于是轮询的,就有可能出现在轮询间隔的时间(这里是1秒)内,有一半时间属于上一根K线,有一半时间属于下一根K线,所以如果我们只看最新的K线的话,前一根K线的一部分数据可能就会丢失,因此这里判断一下前一根K线如果有更新,就先把前一根K线推送了,再推送下一根K线 prev_kline = self._parse_kline(tdx_klines[-2]) latest_kline = self._parse_kline(tdx_klines[-1]) if last_kline and prev_kline['datetime'] == last_kline['datetime']: self.logger.debug('yield prev kline: %s' % prev_kline) yield prev_kline if last_kline != latest_kline: yield latest_kline self.logger.debug('yield latest kline: %s' % latest_kline) last_kline = latest_kline await sleep(1) async def get_kline_histories(self, symbol: str, from_ts: Optional[int] = None, limit: Optional[int] = None): market, code = symbol.split('.') if self._market_mapping[market] in self._market_list: bars = [] if from_ts is not None: idx = 0 while len(bars) == 0 or bars[-1]['datetime'] >= from_ts: bars += [ self._parse_kline(bar) for bar in reversed( self._get_instrument_bars( TDXParams.KLINE_TYPE_1MIN, self._market_mapping[market], code, idx * 700, 700)) ] idx += 1 # 前面都是整700地取,所以有可能取到的数据会超过 from_ts,因此这里再做一次过滤 bars = list(filter(lambda x: x['datetime'] >= from_ts, bars)) elif limit is not None: for i in range(limit // 700): bars += [ self._parse_kline(bar) for bar in reversed( self._get_instrument_bars( TDXParams.KLINE_TYPE_1MIN, self._market_mapping[market], code, i * 700, 700)) ] bars += [ self._parse_kline(bar) for bar in reversed( self._get_instrument_bars(TDXParams.KLINE_TYPE_1MIN, self._market_mapping[market], code, limit // 700 * 700, limit % 700)) ] return reversed(bars) def disconnect(self): self._pill.set() self._api.disconnect() def _get_instrument_bars(self, category, market, code, start=0, count=700): while True: try: cmd = GetInstrumentBars(self._api.client, lock=self._api.lock) cmd.setParams(category, market, code, start=start, count=count) return cmd.call_api() except Exception as e: pass def _parse_kline(self, kline_tdx: dict) -> dict: dt = self._server_tz.localize( datetime.strptime(kline_tdx['datetime'], '%Y-%m-%d %H:%M')).astimezone(pytz.utc) return { 'datetime': int(dt.timestamp() * 1000), 'open': str(round(kline_tdx['open'], 2)), 'high': str(round(kline_tdx['high'], 2)), 'low': str(round(kline_tdx['low'], 2)), 'close': str(round(kline_tdx['close'], 2)), 'volume': kline_tdx['trade'], }
class data(object): def __init__(self, heartbeat=True, isstock=False): self.api = None self.isstock = isstock self.heartbeat = heartbeat self.number = 80000 self.market = None self.datatype = 0 self.result = [] self.TDX_IP_SETS = [ "119.147.86.168", '119.97.185.5' '202.103.36.71', '139,196,185,253', '61.152.107.171', ] # 218.75.74.103:7721 60.12.15.21:7721 self.TDX_IP_SETS_STOCK = [ '119.147.164.60', '218.75.126.9', '115.238.90.165', '124.160.88.183', '60.12.136.250', '218.108.98.244', '218.108.47.69', '14.17.75.71', '180.153.39.51' ] self.file_incon = FILE_INCON self.file_tdxhy = FILE_TDXHY self.file_tdxzs = FILE_TDXZS self.weight = {} def _get_incon(self, ): '''获取行业分类代码 ''' f = open(self.file_incon, "rb") data = f.read() strings = data.decode("gbk", 'ignore').rstrip("\x00").replace("\r\n", "\n") data = strings.split("######") rst = {} for hystr in data: key = re.findall(r'#.*', hystr) if key == ['#TDXNHY']: hylst = hystr.replace("#TDXNHY", "").strip("\n").split("\n") for item in hylst: k, v = item.split("|") rst[k] = [v] return rst def _get_tdxhy(self, islocal=True): '''获取股票和行业对应列表 ''' if islocal: stocklist = HY_WEIGHT.keys() else: stocklist = list(ts.get_stock_basics().index) #获取全市场股票代码 rst = self._get_incon() f = open(self.file_tdxhy, "rb") data = f.read().decode("gbk", 'ignore').rstrip("\x00").replace( "\r\n", "\n").strip("\n").split("\n") for i in data: _, code, tdxhy, _, _ = i.split("|") if tdxhy != "T00" and code in stocklist: rst[tdxhy].append(code) return rst def _get_tdxzs(self, islocal=True): '''生成通达性版块代码对应股票列表 ''' dct = {} rst = self._get_tdxhy(islocal=islocal) f = open(self.file_tdxzs, "rb") data = f.read().decode("gbk", 'ignore').rstrip("\x00").replace( "\r\n", "\n").strip("\n").split("\n") for i in data: name, code, _, _, _, hy = i.split("|") code = int(code) if 880301 <= code and 880497 >= code and hy in rst.keys(): k = hy[:5] if not dct.__contains__(k): dct[k] = {"name": "", "code": "", "stocklist": []} if k == hy: dct[k]["name"] = name dct[k]["code"] = code dct[k]["stocklist"].extend(rst[hy][1:]) return dct def get_tdxhy_list(self, islocal=True): '''获取通达信行业板块指数对应的股票列表 ''' return self._get_tdxzs(islocal) def get_weight(self, htlist={}, islocal=True): '''获取行业板块个股权重,流动市值为权重系数 备注:回测是为方便处理,以最后一天的权重系数作为历史上的权重 ''' if islocal: self.weight = HY_WEIGHT else: if not htlist: htlist = self.get_tdxhy_list(islocal) tasks = [] for v in htlist.values(): tasks.append(gevent.spawn(self.get_latest_ltsz, v["stocklist"])) gevent.joinall(tasks) return self.weight def get_latest_ltsz(self, stocks=[]): '''获取最新流通市值,千万为单位,取整 ''' unit = 10000000 for code in stocks: mk = self._select_market_code(code) print(mk, code) try: ltgb = self.api.get_finance_info(mk, code)["liutongguben"] price = self.api.get_security_bars(4, mk, code, 0, 1)[0]["close"] ltsz = int(ltgb * price / unit) self.weight[code] = ltsz except: print("*****", code) return self.weight @property def mdb(self): '''设置数据库连接 ''' if not hasattr(self, "_db"): self._db = MongoDB() return self._db @property def starttime(self): '''采样起始时间 ''' if not hasattr(self, "_starttime"): self._starttime = "2006-01-01" return self._starttime @starttime.setter def starttime(self, value): self._starttime = value return self._starttime @property def endtime(self): '''采样结束时间 ''' if not hasattr(self, "_endtime"): self._endtime = str(datetime.date.today()) return self._endtime @endtime.setter def endtime(self, value): self._endtime = value return self._endtime def getdata_m( self, collection, db=MIN_STOCK_DB, project={ "open": 1, "high": 1, "low": 1, "close": 1, "datetime": 1, "vol": 1, "_id": 0 }): '''从数据库获取分钟数据 ''' filt = {"datetime": {"$gte": self.starttime, "$lte": self.endtime}} data = [ i for i in self.mdb._dbclient(db)[collection].find(filt, project) ] return pd.DataFrame(data) def getcollections(self, db=MIN_STOCK_DB): data = self.mdb.getallcollections(db) return data def clean_m(self, df, field="datetime"): '''数据库数据有些格式有问题,比如,有些11.30有数据,有些没有。有些13.00有数据,有些没有 导致用分钟数据生成30分钟数据时出现异常 ''' df[field] = df[field].str.replace("13:00", "11:30") df.set_index(field, inplace=True, drop=False) df.index = pd.DatetimeIndex(df.index) df.sort_index(inplace=True) df["time"] = pd.date_range('1/1/2011 00-01-00', periods=df.shape[0], freq='T') return df def createdir(self, path): '''创建输出目录 ''' if not os.path.exists(path): os.makedirs(path) return def setmarket(self): '''设置商品市场代码 ''' market = [] for i in range(100): market += self.api.get_instrument_info(i * 500, 500) self.market = self.api.to_df(market) return self.market def connect(self): if self.isstock: self.api = TdxHq_API(heartbeat=self.heartbeat) port = 7709 TDX_IP_SETS = self.TDX_IP_SETS_STOCK else: self.api = TdxExHq_API(heartbeat=self.heartbeat) port = 7727 TDX_IP_SETS = self.TDX_IP_SETS for ip in TDX_IP_SETS: try: if self.api.connect(ip, port): return except: pass def disconnect(self): self.api.disconnect() def getdata(self, product="ICL8", market=47, number=80000, pn=400): if product[0] in ["0", "3", "6"]: info = self.fetch_get_stock_xdxr(product) data = self.getdata_stock(product, number=number, pn=pn) data.drop(data[data["close"] <= 0].index) df = self.qfq(data, info) elif product[0] in [ "8", ]: df = self.getdata_block_index(product, market, number=number) else: df = self.getdata_future(product, market, number=number) return df def getdata_block_index(self, code="000001", market=1, number=30000, pn=500): data = [] start = False for i in range(int(number / pn) + 1): temp = self.api.get_index_bars(self.datatype, market, code, (int(number / pn) - i) * pn, pn) if temp and len(temp) > 0: start = True if start and (not temp or len(temp) < pn): for _ in range(3): temp = self.api.get_index_bars(self.datatype, market, code, (int(number / pn) - i) * pn, pn) if len(temp) < pn: print(111111111111, pn - len(temp)) else: break try: data += temp except: self.connect() df = self.api.to_df(data)[["open", "close", "high", "low", "datetime"]] df.set_index("datetime", inplace=True, drop=False) return df def getdata_future(self, product="ICL8", market=47, number=80000, pn=400): data = [] start = False for i in range(int(number / pn) + 1): temp = self.api.get_instrument_bars(self.datatype, market, product, (int(number / pn) - i) * pn, pn) if temp and len(temp) > 0: start = True if start and (not temp or len(temp) < pn): for _ in range(3): temp = self.api.get_instrument_bars( self.datatype, market, product, (int(number / pn) - i) * pn, pn) try: if len(temp) < pn: print(111111111111, pn - len(temp)) else: break except: self.connect() try: data += temp except: self.connect() df = self.api.to_df(data)[["open", "close", "high", "low", "datetime"]] df.set_index("datetime", inplace=True, drop=False) return df def set_main_rate(self, df_m, product="RBL8", f="MainContract.csv"): '''商品主月除权 ''' df = pd.read_csv(f, encoding="gb2312") df_p = df[df["ContractCode"] == product[:-2]] lstr = " 15:00" df_p = df_p.assign( datetime=df_p['EndDate'].apply(lambda x: str(x)[0:10] + lstr)) df_p.set_index("datetime", inplace=True) df_p.fillna(0, inplace=True) df_m.loc[:, "date"] = df_m["datetime"].apply(lambda x: str(x)[0:10]) filt = df_m.index.isin(df_p.index) df_m.loc[filt, "OpenPrice"] = df_p["OpenPrice"] df_m.loc[filt, "Term"] = df_p["Term"] df_m["OpenPrice"].fillna(method="bfill", inplace=True) df_m["Term"].fillna(method="bfill", inplace=True) df_m.dropna(inplace=True) filt = (df_m["OpenPrice"]!=df_m["open"])&\ (df_m["date"]>df_m["date"].shift(1))&\ (abs(1-df_m["open"]/df_m["OpenPrice"])>0.008)&\ (df_m["OpenPrice"]>0) df_m.loc[filt, "change"] = 1 df_m.loc[:, "adj"] = 1 rst = df_m[df_m["change"] > 0] rst = (rst["Term"] != rst["Term"].shift(1)) filt = df_m.index.isin(rst[rst > 0].index) df_m.loc[filt, "adj"] = df_m["open"] / df_m["OpenPrice"] df_m.loc[:, 'adj'] = df_m["adj"].shift(-1) df_m.loc[:, 'adj'] = df_m["adj"][::-1].cumprod() print(df_m[(df_m["change"] > 0) | (df_m["change"].shift(-1) > 0)][[ "close", "adj", "open", "OpenPrice", "Term" ]]) df_m.loc[:, 'open'] = df_m['open'] * df_m['adj'] df_m.loc[:, 'high'] = df_m['high'] * df_m['adj'] df_m.loc[:, 'low'] = df_m['low'] * df_m['adj'] df_m.loc[:, 'close'] = df_m['close'] * df_m['adj'] return df_m def getdata_stock(self, code="000001", number=30000, pn=500): market = self._select_market_code(code) data = [] for i in range(int(number / pn) + 1): data += self.api.get_security_bars(self.datatype, market, code, (int(number / pn) - i) * pn, pn) df = self.api.to_df(data)[["open", "close", "high", "low", "datetime"]] df.set_index("datetime", inplace=True, drop=False) return df def _select_market_code(self, code): code = str(code) if code[0] in ['5', '6', '9'] or code[:3] in [ "009", "126", "110", "201", "202", "203", "204" ]: return 1 return 0 def fetch_get_stock_xdxr(self, code): '除权除息' market_code = self._select_market_code(code) category = { '1': '除权除息', '2': '送配股上市', '3': '非流通股上市', '4': '未知股本变动', '5': '股本变化', '6': '增发新股', '7': '股份回购', '8': '增发新股上市', '9': '转配股上市', '10': '可转债上市', '11': '扩缩股', '12': '非流通股缩股', '13': '送认购权证', '14': '送认沽权证' } data = self.api.to_df(self.api.get_xdxr_info(market_code, code)) if len(data) >= 1: data = data\ .assign(date=pd.to_datetime(data[['year', 'month', 'day']]))\ .drop(['year', 'month', 'day'], axis=1)\ .assign(category_meaning=data['category'].apply(lambda x: category[str(x)]))\ .assign(code=str(code))\ .rename(index=str, columns={'panhouliutong': 'liquidity_after', 'panqianliutong': 'liquidity_before', 'houzongguben': 'shares_after', 'qianzongguben': 'shares_before'})\ .set_index('date', drop=False, inplace=False) if self.datatype == 0: lstr = " 09:35" elif self.datatype == 1: lstr = " 09:45" elif self.datatype == 2: lstr = " 10:00" elif self.datatype == 3: lstr = " 10:30" elif self.datatype == 4: lstr = " 15:00" elif self.datatype == 7: lstr = " 09:31" return data.assign( date=data['date'].apply(lambda x: str(x)[0:10] + lstr)) else: return None def qfq(self, data, xdxr_data): '''data: 除权前数据 info:除权信息 ''' start = data.index[0] if xdxr_data is not None: info = xdxr_data[xdxr_data["category"] == 1] info.set_index("date", inplace=True) df = pd.concat( [data, info[['fenhong', 'peigu', 'peigujia', 'songzhuangu']]], axis=1).fillna(0) df['preclose'] = (df['close'].shift(1) * 10 - df['fenhong'] + df['peigu'] * df['peigujia']) / ( 10 + df['peigu'] + df['songzhuangu']) df['adj'] = (df['preclose'].shift(-1) / df['close']).fillna(1)[::-1].cumprod() df['open'] = df['open'] * df['adj'] df['high'] = df['high'] * df['adj'] df['low'] = df['low'] * df['adj'] df['close'] = df['close'] * df['adj'] df['preclose'] = df['preclose'] * df['adj'] else: df['preclose'] = df['close'].shift(1) df['adj'] = 1 return df[start:] def macdhandle(self, df, p5=5, p15=15, p30=30, p60=60, p240=240, pweek=1200, macd_f=12, macd_s=26, macd_m=9): df.loc[:, "number"] = range(df.shape[0]) for i in [p5, p15, p30, p60, p240, pweek]: pres = str(i) + "_" #计算各周期初始macd df.loc[::i, pres + "fEMA_mark"] = df["close"][::i].ewm( adjust=False, span=macd_f).mean() df.loc[::i, pres + "sEMA_mark"] = df["close"][::i].ewm( adjust=False, span=macd_s).mean() df.loc[::i, pres + "DIFF_mark"] = df[pres + "fEMA_mark"] - df[pres + "sEMA_mark"] df[pres + "fEMA_mark"].fillna(method="ffill", inplace=True) df[pres + "sEMA_mark"].fillna(method="ffill", inplace=True) df[pres + "DIFF_mark"].fillna(method="ffill", inplace=True) df[pres + "fEMA"] = (df[pres + "fEMA_mark"] * (macd_f - 1) + df["close"] * 2) / (macd_f + 1) df[pres + "sEMA"] = (df[pres + "sEMA_mark"] * (macd_s - 1) + df["close"] * 2) / (macd_s + 1) df.loc[::i, pres + "fEMA"] = df[pres + "fEMA_mark"] df.loc[::i, pres + "sEMA"] = df[pres + "sEMA_mark"] df.loc[:, pres + "DIFF"] = df[pres + "fEMA"] - df[pres + "sEMA"] df.loc[::i, pres + "DEA_mark"] = df[pres + "DIFF"][::i].ewm( adjust=False, span=macd_m).mean() df[pres + "DEA_mark"].fillna(method="ffill", inplace=True) df[pres + "DEA"] = (df[pres + "DEA_mark"] * (macd_m - 1) + df[pres + "DIFF"] * 2) / (macd_m + 1) df.loc[::i, pres + "DEA"] = df[pres + "DEA_mark"] df.loc[:, pres + "MACD"] = 2 * (df[pres + "DIFF"] - df[pres + "DEA"]) df.loc[:, "DIF"], df.loc[:, "DEA"], df.loc[:, "MACD"] = talib.MACD( df.close.values) df.dropna(inplace=True) #丢弃前面NA数据 return df def getblockstock(self, block="沪深300"): '''股票版块对应股票列表 ''' df = self.api.to_df(self.api.get_and_parse_block_info("block.dat")) stocks = list(df[df["blockname"] == block]["code"]) return stocks
''' tdx扩展行情接口API https://rainx.gitbooks.io/pytdx/content/pytdx_exhq.html ''' #首先需要引入 from pytdx.exhq import TdxExHq_API #然后,创建对象 api = TdxExHq_API() #连接行情服务器 api.connect('106.14.95.149', 7727, time_out=30) #获取市场代码 df=api.to_df(api.get_markets()) #输出信息 print(df) df2=api.get_history_minute_time_data(31, "00020", 20170811) df3=api.to_df(df2) print(df3) #断开连接 api.disconnect()