def QA_fetch_get_future_day(code, start_date, end_date, frequence='day', ip=best_ip['future'], port=7727): '期货数据 日线' apix = TdxExHq_API() start_date = str(start_date)[0:10] today_ = datetime.date.today() lens = QA_util_get_trade_gap(start_date, today_) global extension_market_info extension_market_info = QA_fetch_get_future_list( ) if extension_market_info is None else extension_market_info with apix.connect(ip, port): code_market = extension_market_info.query('code=="{}"'.format(code)) data = pd.concat([ apix.to_df( apix.get_instrument_bars(_select_type(frequence), int(code_market.market), str(code), (int(lens / 700) - i) * 700, 700)) for i in range(int(lens / 700) + 1) ], axis=0) data = data.assign(date=data['datetime'].apply(lambda x: str(x[0:10]))).assign(code=str(code))\ .assign(date_stamp=data['datetime'].apply(lambda x: QA_util_date_stamp(str(x)[0:10]))).set_index('date', drop=False, inplace=False) return data.drop( ['year', 'month', 'day', 'hour', 'minute', 'datetime'], axis=1)[start_date:end_date].assign( date=data['date'].apply(lambda x: str(x)[0:10]))
def get_instrument_info(): api = TdxExHq_API(heartbeat=True) host = "180.153.18.176" #通信达的api地址 port = 7721 #通信达的连接端口 #开始连接通信达服务器 api.connect(host, port) insts = [] count = 500 curr_index = 0 while (True): insts_tmp = api.get_instrument_info(curr_index, count) if insts_tmp is None: break insts.extend(insts_tmp) curr_index = curr_index + len(insts_tmp) if len(insts_tmp) < count: break #查看通信达提供的市场列表 #print api.to_df(api.get_markets()) df_inst = api.to_df(insts) #这里笔者选择的美国知名公司列表, 所以market = 41 df_inst[df_inst['market'] == 41] #这里教程获取AAPL单一数据,如果需要全部数据可以使用列表循环下载整个数据 #笔者获取的是苹果公司最近300天交易日行情Day版 his_kline = api.get_instrument_bars(TDXParams.KLINE_TYPE_DAILY, 41, "AAPL", 0, 300) datadf = api.to_df(his_kline) #保存为csv格式,命名为APPL-demo/按照逗号分隔 datadf.to_csv(os.getcwd() + "/tdx_file/" + 'APPL-demo.csv', index=False, sep=',')
def QA_fetch_get_future_day(code, start_date, end_date, frequence='day', ip=None, port=None): '期货数据 日线' global best_ip if ip is None and port is None and best_ip['future']['ip'] is None and best_ip['future']['port'] is None: best_ip = select_best_ip() ip = best_ip['future']['ip'] port = best_ip['future']['port'] elif ip is None and port is None and best_ip['future']['ip'] is not None and best_ip['future']['port'] is not None: ip = best_ip['future']['ip'] port = best_ip['future']['port'] else: pass apix = TdxExHq_API() start_date = str(start_date)[0:10] today_ = datetime.date.today() lens = QA_util_get_trade_gap(start_date, today_) global extension_market_info extension_market_info = QA_fetch_get_future_list( ) if extension_market_info is None else extension_market_info with apix.connect(ip, port): code_market = extension_market_info.query('code=="{}"'.format(code)) data = pd.concat([apix.to_df(apix.get_instrument_bars(_select_type( frequence), int(code_market.market), str(code), (int(lens / 700) - i) * 700, 700))for i in range(int(lens / 700) + 1)], axis=0) data = data.assign(date=data['datetime'].apply(lambda x: str(x[0:10]))).assign(code=str(code))\ .assign(date_stamp=data['datetime'].apply(lambda x: QA_util_date_stamp(str(x)[0:10]))).set_index('date', drop=False, inplace=False) return data.drop(['year', 'month', 'day', 'hour', 'minute', 'datetime'], axis=1)[start_date:end_date].assign(date=data['date'].apply(lambda x: str(x)[0:10]))
def QA_fetch_get_future_min(code, start, end, frequence='1min', ip=best_ip['future'], port=7727): '期货数据 分钟线' apix = TdxExHq_API() type_ = '' start_date = str(start)[0:10] today_ = datetime.date.today() lens = QA_util_get_trade_gap(start_date, today_) global extension_market_info extension_market_info = QA_fetch_get_future_list( ) if extension_market_info is None else extension_market_info if str(frequence) in ['5', '5m', '5min', 'five']: frequence, type_ = 0, '5min' lens = 48 * lens elif str(frequence) in ['1', '1m', '1min', 'one']: frequence, type_ = 8, '1min' lens = 240 * lens elif str(frequence) in ['15', '15m', '15min', 'fifteen']: frequence, type_ = 1, '15min' lens = 16 * lens elif str(frequence) in ['30', '30m', '30min', 'half']: frequence, type_ = 2, '30min' lens = 8 * lens elif str(frequence) in ['60', '60m', '60min', '1h']: frequence, type_ = 3, '60min' lens = 4 * lens if lens > 20800: lens = 20800 with apix.connect(ip, port): code_market = extension_market_info.query('code=="{}"'.format(code)) data = pd.concat([ apix.to_df( apix.get_instrument_bars(frequence, int(code_market.market), str(code), (int(lens / 700) - i) * 700, 700)) for i in range(int(lens / 700) + 1) ], axis=0) data = data\ .assign(datetime=pd.to_datetime(data['datetime']), code=str(code))\ .drop(['year', 'month', 'day', 'hour', 'minute'], axis=1, inplace=False)\ .assign(date=data['datetime'].apply(lambda x: str(x)[0:10]))\ .assign(date_stamp=data['datetime'].apply(lambda x: QA_util_date_stamp(x)))\ .assign(time_stamp=data['datetime'].apply(lambda x: QA_util_time_stamp(x)))\ .assign(type=type_).set_index('datetime', drop=False, inplace=False)[start:end] return data.assign(datetime=data['datetime'].apply(lambda x: str(x)))
def QA_fetch_get_future_min(code, start, end, frequence='1min', ip=None, port=None): '期货数据 分钟线' global best_ip if ip is None and port is None and best_ip['future']['ip'] is None and best_ip['future']['port'] is None: best_ip = select_best_ip() ip = best_ip['future']['ip'] port = best_ip['future']['port'] elif ip is None and port is None and best_ip['future']['ip'] is not None and best_ip['future']['port'] is not None: ip = best_ip['future']['ip'] port = best_ip['future']['port'] else: pass apix = TdxExHq_API() type_ = '' start_date = str(start)[0:10] today_ = datetime.date.today() lens = QA_util_get_trade_gap(start_date, today_) global extension_market_info extension_market_info = QA_fetch_get_future_list( ) if extension_market_info is None else extension_market_info if str(frequence) in ['5', '5m', '5min', 'five']: frequence, type_ = 0, '5min' lens = 48 * lens elif str(frequence) in ['1', '1m', '1min', 'one']: frequence, type_ = 8, '1min' lens = 240 * lens elif str(frequence) in ['15', '15m', '15min', 'fifteen']: frequence, type_ = 1, '15min' lens = 16 * lens elif str(frequence) in ['30', '30m', '30min', 'half']: frequence, type_ = 2, '30min' lens = 8 * lens elif str(frequence) in ['60', '60m', '60min', '1h']: frequence, type_ = 3, '60min' lens = 4 * lens if lens > 20800: lens = 20800 with apix.connect(ip, port): code_market = extension_market_info.query('code=="{}"'.format(code)) data = pd.concat([apix.to_df(apix.get_instrument_bars(frequence, int(code_market.market), str( code), (int(lens / 700) - i) * 700, 700)) for i in range(int(lens / 700) + 1)], axis=0) data = data\ .assign(datetime=pd.to_datetime(data['datetime']), code=str(code))\ .drop(['year', 'month', 'day', 'hour', 'minute'], axis=1, inplace=False)\ .assign(date=data['datetime'].apply(lambda x: str(x)[0:10]))\ .assign(date_stamp=data['datetime'].apply(lambda x: QA_util_date_stamp(x)))\ .assign(time_stamp=data['datetime'].apply(lambda x: QA_util_time_stamp(x)))\ .assign(type=type_).set_index('datetime', drop=False, inplace=False)[start:end] return data.assign(datetime=data['datetime'].apply(lambda x: str(x)))
def get_option_daily(code_list): """ 从pytdx模块中获取期权价格。 :param code_list: :return: """ api = TdxExHq_API() df_list = [] with api.connect(ip=PYTDX_EXHQ_SERVER): for code in code_list: df = api.to_df( api.get_instrument_bars(TDXParams.KLINE_TYPE_DAILY, 8, code, 0, 100)) df_list.append(df) df_merge = pd.concat(df_list) df_merge.to_csv('pytdx_price.csv', encoding=TA_CSV_CODING) return df_merge
def __exhq_bars(self, code, offset, frequency=9): assert self.__ex.qsize() > 0 api = TdxExHq_API() ip, port = self.__ex.get() with api.connect(ip, port): df = list() for _code in code: for _start, _count in offset: df.append( api.to_df( api.get_instrument_bars( frequency, self.__exhq_list().xs(_code).market, _code, _start, _count)).assign(code=_code)) api.disconnect() self.__ex.put((ip, port)) if len(df) < 1: return None return pandas.concat(df, sort=False)
class Datacent: def __init__(self): self.cf = config self.qihuo_api = TdxExHq_API() self.cflen = 0 def qihuo_connectSer(self): self.qihuo_api.connect('218.80.248.229', 7721) qihuoret = self.qihuo_api.connect('218.80.248.229', 7721) if qihuoret == False: print("期货没有连接。。。") return qihuoret else: print("已连接期货数据服务") qihuoret = True return qihuoret def qihuoK( self, cflen, ): data = self.qihuo_api.get_instrument_bars( self.cf.category, int(self.cf.cfqihuo[cflen]["marketid"]), self.cf.cfqihuo[cflen]['code'], 0, self.cf.categorycount) #7: 扩展行情查询k线数据 df = pd.DataFrame( data, columns=[ 'datetime', 'stockname', 'open', 'high', 'low', 'close', 'code', ], ) df['stockname'] = self.cf.cfqihuo[cflen]["stockname"] df['code'] = self.cf.cfqihuo[cflen]["code"] return df
class ExtQuotes(object): """扩展市场实时行情""" config = None client = None bestip = ('112.74.214.43', 7727) def __init__(self, **kwargs): try: default = settings.get('SERVER').get('EX')[0] self.bestip = config.get('BESTIP').get('EX', default) except ValueError: self.bestip = ('112.74.214.43', 7727) self.client = TdxExHq_API(**kwargs) def validate(self, market, symbol): if not market: if len(symbol.split('#')) > 1: market = symbol.split('#')[0] symbol = symbol.split('#')[1] if not market: raise ValueError('市场参数错误, 市场参数不能为空.') return int(market), symbol def markets(self): ''' 获取实时市场列表 :return: pd.dataFrame or None ''' with self.client.connect(*self.bestip) as client: result = client.get_markets() return to_data(result) def instrument(self, start=0, offset=100): ''' 查询代码列表 :param offset: :param start: :return: ''' with self.client.connect(*self.bestip): result = self.client.get_instrument_info(start=start, count=offset) return to_data(result) def instrument_count(self): ''' 市场商品数量 :return: ''' with self.client.connect(*self.bestip): result = self.client.get_instrument_count() return result def instruments(self): ''' 查询所有代码列表 :return: ''' with self.client.connect(*self.bestip): count = self.client.get_instrument_count() pages = math.ceil(count / 100) result = [] for page in tqdm(range(0, pages)): result += self.client.get_instrument_info( page * 100, (page + 1) * 100) return to_data(result) def quote(self, market='', symbol=''): ''' 查询五档行情 :param market: :param symbol: :return: ''' market, symbol = self.validate(market, symbol) with self.client.connect(*self.bestip): result = self.client.get_instrument_quote(market, symbol) return to_data(result) def minute(self, market='', symbol=''): ''' 查询分时行情 :param market: :param symbol: :return: ''' market, symbol = self.validate(market, symbol) with self.client.connect(*self.bestip): result = self.client.get_minute_time_data(market, symbol) return to_data(result) def minutes(self, market=None, symbol='', date=''): ''' 查询历史分时行情 :param market: :param symbol: :param date: :return: ''' market, symbol = self.validate(market, symbol) with self.client.connect(*self.bestip): result = self.client.get_history_minute_time_data( market, symbol, date) return to_data(result) def bars(self, frequency='', market='', symbol='', start='', offset=0): ''' 查询k线数据 参数: K线周期, 市场ID, 证券代码,起始位置, 数量 :param frequency: K线周期 :param market: 市场ID :param symbol: 证券代码 :param start: 起始位置 :param offset: 数量 :return: ''' market, symbol = self.validate(market, symbol) with self.client.connect(*self.bestip): result = self.client.get_instrument_bars(frequency=frequency, market=market, code=symbol, start=start, count=offset) return to_data(result) def transaction(self, market=None, symbol='', start=0, offset=1800): ''' 查询分笔成交 :param market: :param symbol: :param start: :param offset: :return: ''' market, symbol = self.validate(market, symbol) with self.client.connect(*self.bestip): result = self.client.get_transaction_data(market=market, code=symbol, start=start, count=offset) return to_data(result) def transactions(self, market=None, symbol='', date='', start=0, offset=1800): ''' 查询历史分笔成交 :param market: :param symbol: :param date: :param start: :param offset: :return: ''' market, symbol = self.validate(market, symbol) with self.client.connect(*self.bestip): result = self.client.get_history_transaction_data(market=market, code=symbol, date=date, start=start, count=offset) return to_data(result)
df["rsi%s" % i] = ta.momentum.rsi(df.close,n=self.step,) return df def stoch_Multi(self): for i in list(range(0,self.step,1)): df["stoch%s" % i] = ta.momentum.stoch(df.high,df.low,df.close,self.step) return df def tradeaction(self): ret = df[df.superpoint!=0].reset_index() ret['returns'] = np.abs(ret.close.shift(periods=1,axis=0) - ret.close)#算zigzag的回报 return ret.round(1) if __name__ == "__main__": api = TdxExHq_API() with api.connect('218.80.248.229', 7721): data = api.get_instrument_bars(cf.category, int(cf.cfqihuo[cflen]["marketid"]), cf.cfqihuo[cflen]['code'], 0, cf.categorycount)#7: 扩展行情查询k线数据 df = pd.DataFrame(data,columns=['datetime', 'open', 'high', 'low', 'close','trade']) df = tdx_tools.SuperTrend(df,period=st_period,multiplier=st_mult,ohlc=['open', 'high', 'low', 'close']) md = makeData(df,50) # md.rsi_Multi() # md.stoch_Multi() ret = md.tradeaction() print (ret) #print (df) ''' api = TdxExHq_API()
kline = OrderedDict([ ("open", open_price), ("high", high), ("low", low), ("close", close), ("amount", amount), ("trade", trade), ("price", price), ("year", year), ("month", month), ("day", day), ("hour", hour), ("minute", minute), ("datetime", "%d-%02d-%02d %02d:%02d" % (year, month, day, hour, minute)) ]) klines.append(kline) return klines if __name__ == '__main__': from pytdx.exhq import TdxExHq_API from pytdx.params import TDXParams api = TdxExHq_API() # cmd = GetInstrumentBars(api) # cmd.setParams(4, 7, "10000843", 0, 10) # print(cmd.send_pkg) with api.connect('61.152.107.141', 7727): print(api.to_df(api.get_instrument_bars(TDXParams.KLINE_TYPE_EXHQ_1MIN, 74, 'BABA')).tail())
# -*- coding: utf-8 -*- from pytdx.exhq import TdxExHq_API import ta api = TdxExHq_API() with api.connect('124.74.236.94', 7721): #查询市场中商品数量 marketsID = api.to_df(api.get_markets()) #查询五档行情 quote =api.to_df(api.get_instrument_quote(47, "IFL8")) #查询分时行情 markets_fenshi = api.to_df(api.get_minute_time_data(47, "IFL8")) #查询历史分时行情 historydata = api.to_df(api.get_history_minute_time_data(47, "IFL8", 20170811)) #查询k线数据 kdata = api.to_df(api.get_instrument_bars(0, 47, "IFL8", 0, 700)) #查询当前分笔成交 fenbi = api.to_df(api.get_transaction_data(47, "IFL8")) #查询历史分笔成交 fenbi_history = api.to_df(api.get_history_transaction_data(47, "IFL8", 20191204, start=1800)) #查询当天分笔成交 fenbi_chenjiao = api.to_df(api.get_history_transaction_data(47, "IFL8", 20191204,)) print (marketsID) print (quote) print (kdata) df = kdata
("day", day), ("hour", hour), ("minute", minute), ("datetime", "%d-%02d-%02d %02d:%02d" % (year, month, day, hour, minute)), ("amount", amount), ]) klines.append(kline) return klines if __name__ == '__main__': from pytdx.exhq import TdxExHq_API from pytdx.params import TDXParams api = TdxExHq_API() # cmd = GetInstrumentBars(api) # cmd.setParams(4, 7, "10000843", 0, 10) # print(cmd.send_pkg) with api.connect('61.152.107.141', 7727): print( api.to_df( api.get_instrument_bars(TDXParams.KLINE_TYPE_EXHQ_1MIN, 74, 'BABA')).tail()) print( api.to_df( api.get_instrument_bars(TDXParams.KLINE_TYPE_DAILY, 31, '00001')).tail())
def update_futures(args): """Update Future 1min data in MongoDB Args: Returns: """ client = pymongo.MongoClient(args.mongo_uri, serverSelectionTimeoutMS=1000) client.server_info() db = client[args.database] collection = db['future_china_1min'] api = TdxExHq_API(heartbeat=True, multithread=True) api.connect('61.152.107.141', 7727) num = api.get_instrument_count() insts = [api.get_instrument_info(i, QSIZE) for i in range(0, num, QSIZE)] insts = [x for i in insts for x in i] exchs = ['中金所期货', '上海期货', '大连商品', '郑州商品'] markets = [t['market'] for t in api.get_markets() if t['name'] in exchs] futures = [t for t in insts if t['market'] in markets and t['code'][-2] != 'L'] for future in futures: qeury = collection.find({"ticker": future['code']}) qeury = qeury.sort('datetime', pymongo.DESCENDING) qeury = qeury.limit(1) last_one = list(qeury) if len(last_one) > 0: last_date = last_one[0]['datetime'] + timedelta(minutes=1) else: last_date = datetime.now() - timedelta(days=365) end_date = datetime.now().date() end_date = datetime.combine(end_date - timedelta(days=1), time(ALL_MARKET_END_HOUR, 0)) _start_date = end_date _bars = [] _pos = 0 while _start_date > last_date: _res = api.get_instrument_bars( TDXParams.KLINE_TYPE_1MIN, future['market'], future['code'], _pos, QSIZE) try: _bars = _res + _bars except TypeError: continue _pos += QSIZE if len(_res) > 0: _start_date = _res[0]['datetime'] _start_date = datetime.strptime(_start_date, '%Y-%m-%d %H:%M') else: break if len(_bars) == 0: continue data = api.to_df(_bars) data = data.assign(datetime=pd.to_datetime(data['datetime'])) data = data.assign(ticker=future['code']) data = data.drop( ['year', 'month', 'day', 'hour', 'minute', 'price', 'amount'], errors='ignore', axis=1) data = data.rename( index=str, columns={ 'position': 'oi', 'trade': 'volume', }) data['date'] = pd.to_datetime(data['datetime'].dt.date) _miss_ts = data['datetime'].dt.hour > ALL_MARKET_END_HOUR data.loc[_miss_ts, 'datetime'] -= timedelta(days=1) data = data.set_index('datetime', drop=False) data = data[str(last_date):str(end_date)] collection.insert_many(data.to_dict('records')) if len(data) > 0 else 0 _logger.info(future['code']) api.disconnect()
class TdxFutureData(object): # ---------------------------------------------------------------------- def __init__(self, strategy=None, best_ip={}): """ 构造函数 :param strategy: 上层策略,主要用与使用write_log() """ self.api = None self.connection_status = False # 连接状态 self.best_ip = best_ip self.symbol_exchange_dict = {} # tdx合约与vn交易所的字典 self.symbol_market_dict = copy.copy( INIT_TDX_MARKET_MAP) # tdx合约与tdx市场的字典 self.strategy = strategy self.future_contracts = get_future_contracts() def write_log(self, content): if self.strategy: self.strategy.write_log(content) else: print(content) def write_error(self, content): if self.strategy: self.strategy.write_log(content, level=ERROR) else: print(content) def connect(self, is_reconnect=False): """ 连接API :return: """ # 创建api连接对象实例 try: if self.api is None or not self.connection_status: self.write_log(u"开始连接通达信行情服务器") self.api = TdxExHq_API(heartbeat=True, auto_retry=True, raise_exception=True) # 选取最佳服务器 if is_reconnect or len(self.best_ip) == 0: self.best_ip = get_cache_json(TDX_FUTURE_CONFIG) last_datetime_str = self.best_ip.get("datetime", None) if last_datetime_str: try: last_datetime = datetime.strptime( last_datetime_str, "%Y-%m-%d %H:%M:%S") if (datetime.now() - last_datetime ).total_seconds() > 60 * 60 * 2: self.best_ip = {} except Exception as ex: self.best_ip = {} else: self.best_ip = {} if len(self.best_ip) == 0: self.best_ip = self.select_best_ip() self.api.connect(self.best_ip["ip"], self.best_ip["port"]) # 尝试获取市场合约统计 c = self.api.get_instrument_count() if c < 10: err_msg = u"该服务器IP {}/{}无响应".format( self.best_ip["ip"], self.best_ip["port"]) self.write_error(err_msg) else: self.write_log(u"创建tdx连接, IP: {}/{}".format( self.best_ip["ip"], self.best_ip["port"])) # print(u"创建tdx连接, IP: {}/{}".format(self.best_ip["ip"], self.best_ip["port"])) self.connection_status = True # if not is_reconnect: # 更新 symbol_exchange_dict , symbol_market_dict # self.qryInstrument() except Exception as ex: self.write_log(u"连接服务器tdx异常:{},{}".format(str(ex), traceback.format_exc())) return # ---------------------------------------------------------------------- def ping(self, ip, port=7709): """ ping行情服务器 :param ip: :param port: :param type_: :return: """ apix = TdxExHq_API() __time1 = datetime.now() try: with apix.connect(ip, port): if apix.get_instrument_count() > 10000: _timestamp = datetime.now() - __time1 self.write_log("服务器{ip}:{port},耗时:{_timestamp}") return _timestamp else: self.write_log("该服务器IP {ip}无响应") return timedelta(9, 9, 0) except Exception: self.write_error("tdx ping服务器,异常的响应{ip}") return timedelta(9, 9, 0) # ---------------------------------------------------------------------- def select_best_ip(self): """ 选择行情服务器 :return: """ self.write_log(u"选择通达信行情服务器") data_future = [self.ping(x["ip"], x["port"]) for x in TDX_FUTURE_HOSTS] best_future_ip = TDX_FUTURE_HOSTS[data_future.index(min(data_future))] self.write_log(u"选取 {}:{}".format(best_future_ip["ip"], best_future_ip["port"])) # print(u"选取 {}:{}".format(best_future_ip["ip"], best_future_ip["port"])) best_future_ip.update( {"datetime": datetime.now().strftime("%Y-%m-%d %H:%M:%S")}) save_cache_json(best_future_ip, TDX_FUTURE_CONFIG) return best_future_ip def _get_vn_exchange(self, symbol): """获取""" underlying_symbol = get_underlying_symbol(symbol).upper() info = self.future_contracts.get(underlying_symbol, None) if info: return Exchange(info.get("exchange")) else: market_id = get_tdx_marketid(symbol) return Tdx_Vn_Exchange_Map.get(str(market_id), Exchange.INE) def qry_instrument(self): """ 查询/更新合约信息 :return: """ # 取得所有的合约信息 num = self.api.get_instrument_count() if not isinstance(num, int): return all_contacts = sum([ self.api.get_instrument_info((int(num / 500) - i) * 500, 500) for i in range(int(num / 500) + 1) ], []) # [{"category":category,"market": int,"code":sting,"name":string,"desc":string},{}] # 对所有合约处理,更新字典 指数合约-tdx市场,指数合约-交易所 for tdx_contract in all_contacts: tdx_symbol = tdx_contract.get("code", None) if tdx_symbol is None: continue tdx_market_id = tdx_contract.get("market") if str(tdx_market_id) in Tdx_Vn_Exchange_Map: self.symbol_exchange_dict.update( {tdx_symbol: Tdx_Vn_Exchange_Map.get(str(tdx_market_id))}) self.symbol_market_dict.update({tdx_symbol: tdx_market_id}) # ---------------------------------------------------------------------- def get_bars(self, symbol, period, callback=None, bar_freq=1, start_dt=None, end_dt=None, return_bar=True): """ 返回k线数据 symbol:合约 period: 周期: 1min,3min,5min,15min,30min,1day,3day,1hour,2hour,4hour,6hour,12hour callback: 逐一bar去驱动回调函数, 只有 return_bar = True时才回调 bar_freq: 回调时的参数 start_dt: 取数据的开始时间 end_dt: 取数据的结束时间 return_bar: 返回 第二个数据内容,True:BarData, False:dict """ ret_bars = [] tdx_symbol = symbol.upper().replace("_", "") tdx_symbol = tdx_symbol.replace("99", "L9") underlying_symbol = get_underlying_symbol(symbol).upper() tdx_index_symbol = underlying_symbol + "L9" vn_exchange = self._get_vn_exchange(underlying_symbol) self.connect() if self.api is None: return False, ret_bars if period not in PERIOD_MAPPING.keys(): self.write_error(u"{} 周期{}不在下载清单中: {}".format( datetime.now(), period, list(PERIOD_MAPPING.keys()))) return False, ret_bars tdx_period = PERIOD_MAPPING.get(period) if start_dt is None: self.write_log(u"没有设置开始时间,缺省为10天前") qry_start_date = datetime.now() - timedelta(days=10) else: qry_start_date = start_dt if end_dt is None: self.write_log(u"没有设置结束时间,缺省为当日") end_date = datetime.combine(datetime.now() + timedelta(days=1), time(ALL_MARKET_END_HOUR, 0)) else: end_date = end_dt if qry_start_date > end_date: qry_start_date = end_date self.write_log("{}开始下载tdx:{} {}数据, {} to {}.".format( datetime.now(), tdx_symbol, period, qry_start_date, end_date)) # print("{}开始下载tdx:{} {}数据, {} to {}.".format(datetime.now(), tdx_symbol, tdx_period, last_date, end_date)) try: _start_date = end_date _bars = [] _pos = 0 while _start_date > qry_start_date: _res = self.api.get_instrument_bars( tdx_period, self.symbol_market_dict.get(tdx_index_symbol, 0), tdx_symbol, _pos, QSIZE) if _res is not None: _bars = _res + _bars _pos += QSIZE if _res is not None and len(_res) > 0: _start_date = _res[0]["datetime"] _start_date = datetime.strptime(_start_date, "%Y-%m-%d %H:%M") self.write_log(u"分段取数据开始:{}".format(_start_date)) else: break if len(_bars) == 0: self.write_error("{} Handling {}, len1={}..., continue".format( str(datetime.now()), tdx_symbol, len(_bars))) return False, ret_bars current_datetime = datetime.now() data = self.api.to_df(_bars) data = data.assign(datetime=to_datetime(data["datetime"])) data = data.assign(ticker=symbol) data["instrument_id"] = data["ticker"] data["symbol"] = symbol data = data.drop([ "year", "month", "day", "hour", "minute", "price", "amount", "ticker" ], errors="ignore", axis=1) data = data.rename(index=str, columns={ "position": "open_interest", "trade": "volume", }) if len(data) == 0: print("{} Handling {}, len2={}..., continue".format( str(datetime.now()), tdx_symbol, len(data))) return False, ret_bars data["total_turnover"] = data["volume"] * data["close"] data["limit_down"] = 0 data["limit_up"] = 999999 data["trading_day"] = data["datetime"] data["trading_day"] = data["trading_day"].apply( lambda x: (x.strftime("%Y-%m-%d"))) monday_ts = data["datetime"].dt.weekday == 0 # 星期一 night_ts1 = data["datetime"].dt.hour > ALL_MARKET_END_HOUR night_ts2 = data["datetime"].dt.hour < ALL_MARKET_BEGIN_HOUR data.loc[night_ts1, "datetime"] -= timedelta( days=1) # 所有日期的夜盘(21:00~24:00), 减一天 monday_ts1 = monday_ts & night_ts1 # 星期一的夜盘(21:00~24:00), 再减两天 data.loc[monday_ts1, "datetime"] -= timedelta(days=2) monday_ts2 = monday_ts & night_ts2 # 星期一的夜盘(00:00~04:00), 再减两天 data.loc[monday_ts2, "datetime"] -= timedelta(days=2) # data["datetime"] -= timedelta(minutes=1) # 直接给Strategy使用, RiceQuant格式, 不需要减1分钟 # data["dt_datetime"] = data["datetime"] data["date"] = data["datetime"].apply(lambda x: (x.strftime("%Y-%m-%d"))) data["time"] = data["datetime"].apply(lambda x: (x.strftime("%H:%M:%S"))) # data["datetime"] = data["datetime"].apply(lambda x: float(x.strftime("%Y%m%d%H%M%S"))) data = data.set_index("datetime", drop=False) if return_bar: self.write_log("dataframe => [bars]") for index, row in data.iterrows(): add_bar = BarData(gateway_name="tdx", symbol=symbol, exchange=vn_exchange, datetime=index) try: add_bar.date = row["date"] add_bar.time = row["time"] add_bar.trading_day = row["trading_day"] add_bar.open_price = float(row["open"]) add_bar.high_price = float(row["high"]) add_bar.low_price = float(row["low"]) add_bar.close_price = float(row["close"]) add_bar.volume = float(row["volume"]) add_bar.openInterest = float(row["open_interest"]) except Exception as ex: self.write_error( "error when convert bar:{},ex:{},t:{}".format( row, str(ex), traceback.format_exc())) # print("error when convert bar:{},ex:{},t:{}".format(row, str(ex), traceback.format_exc())) return False, ret_bars if start_dt is not None and index < start_dt: continue ret_bars.append(add_bar) if callback is not None: freq = bar_freq bar_is_completed = True if period != "1min" and index == data["datetime"][-1]: # 最后一个bar,可能是不完整的,强制修改 # - 5min修改后freq基本正确 # - 1day在VNPY合成时不关心已经收到多少Bar, 所以影响也不大 # - 但其它分钟周期因为不好精确到每个品种, 修改后的freq可能有错 if index > current_datetime: bar_is_completed = False # 根据秒数算的话,要+1,例如13:31,freq=31,第31根bar freq = NUM_MINUTE_MAPPING[period] - int( (index - current_datetime).total_seconds() / 60) callback(add_bar, bar_is_completed, freq) else: self.write_log("dataframe => [ dict ]") ret_bars = list(data.T.to_dict().values()) return True, ret_bars except Exception as ex: self.write_error("exception in get:{},{},{}".format( tdx_symbol, str(ex), traceback.format_exc())) # print("exception in get:{},{},{}".format(tdx_symbol,str(ex), traceback.format_exc())) self.write_log(u"重置连接") self.api = None self.connect(is_reconnect=True) return False, ret_bars def get_price(self, symbol): """获取最新价格""" tdx_symbol = symbol.upper().replace("_", "") short_symbol = get_underlying_symbol(tdx_symbol).upper() if tdx_symbol.endswith("99"): query_symbol = tdx_symbol.replace("99", "L9") else: query_symbol = get_full_symbol(tdx_symbol) if query_symbol != tdx_symbol: self.write_log("转换合约:{}=>{}".format(tdx_symbol, query_symbol)) tdx_index_symbol = short_symbol + "L9" self.connect() if self.api is None: return 0 market_id = self.symbol_market_dict.get(tdx_index_symbol, 0) _res = self.api.get_instrument_quote(market_id, query_symbol) if not isinstance(_res, list): return 0 if len(_res) == 0: return 0 return float(_res[0].get("price", 0)) def get_99_contracts(self): """ 获取指数合约 :return: dict list """ self.connect() result = self.api.get_instrument_quote_list(42, 3, 0, 100) return result def get_mi_contracts(self): """ 获取主力合约 :return: dict list """ self.connect() result = self.api.get_instrument_quote_list(60, 3, 0, 100) return result def get_contracts(self, exchange): self.connect() market_id = Vn_Tdx_Exchange_Map.get(exchange, None) if market_id is None: print(u"市场:{}配置不在Vn_Tdx_Exchange_Map:{}中,不能取市场下所有合约".format( exchange, Vn_Tdx_Exchange_Map)) return [] index = 0 count = 100 results = [] while (True): print(u"查询{}下:{}~{}个合约".format(exchange, index, index + count)) #print("helloabc",self.api.get_instrument_quote_list(47,"IF2003")) result = self.api.get_instrument_quote_list( int(market_id), 3, index, count) results.extend(result) index += count if len(result) < count: break return results def get_mi_contracts2(self): """ 获取主力合约""" self.connect() contracts = [] for exchange in Vn_Tdx_Exchange_Map.keys(): contracts.extend(self.get_mi_contracts_from_exchange(exchange)) return contracts def get_mi_contracts_from_exchange(self, exchange): contracts = self.get_contracts(exchange) if len(contracts) == 0: print(u"异常,未能获取{}下合约信息".format(exchange)) return [] mi_contracts = [] short_contract_dict = {} for contract in contracts: # 排除指数合约 code = contract.get("code") if code[-2:] in ["L9", "L8", "L0", "L1", "L2", "L3", "50"] or \ (exchange == Exchange.CFFEX and code[-3:] in ["300", "500"]): continue short_symbol = get_underlying_symbol(code).upper() contract_list = short_contract_dict.get(short_symbol, []) contract_list.append(contract) short_contract_dict.update({short_symbol: contract_list}) for k, v in short_contract_dict.items(): sorted_list = sorted(v, key=lambda c: c["ZongLiang"]) mi_contracts.append(sorted_list[-1]) return mi_contracts def get_markets(self): """ 获取市场代码 :return: """ self.connect() result = self.api.get_markets() return result def get_transaction_data(self, symbol): """获取当前交易日的历史成交记录""" ret_datas = [] max_data_size = sys.maxsize symbol = symbol.upper() if "99" in symbol: # 查询的是指数合约 symbol = symbol.replace("99", "L9") tdx_index_symbol = symbol else: # 查询的是普通合约 tdx_index_symbol = get_underlying_symbol(symbol).upper() + "L9" self.connect() q_size = QSIZE * 5 # 每秒 2个, 10小时 max_data_size = 1000000 self.write_log(u"开始下载{}当日分笔数据".format(symbol)) try: _datas = [] _pos = 0 while True: _res = self.api.get_transaction_data( market=self.symbol_market_dict.get(tdx_index_symbol, 0), code=symbol, start=_pos, count=q_size) if _res is not None: for d in _res: dt = d.pop("date") # 星期1~星期6 if dt.hour >= 20 and 1 < dt.isoweekday() <= 6: dt = dt - timedelta(days=1) elif dt.hour >= 20 and dt.isoweekday() == 1: # 星期一取得20点后数据 dt = dt - timedelta(days=3) elif dt.hour < 8 and dt.isoweekday() == 1: # 星期一取得8点前数据 dt = dt - timedelta(days=3) elif dt.hour >= 20 and dt.isoweekday() == 7: # 星期天取得20点后数据,肯定是星期五夜盘 dt = dt - timedelta(days=2) elif dt.isoweekday() == 7: # 星期日取得其他时间,必然是 星期六凌晨的数据 dt = dt - timedelta(days=1) d.update({"datetime": dt}) # 接口有bug,返回价格*1000,所以要除以1000 d.update({"price": d.get("price", 0) / 1000}) _datas = sorted(_res, key=lambda s: s["datetime"]) + _datas _pos += min(q_size, len(_res)) if _res is not None and len(_res) > 0: self.write_log(u"分段取分笔数据:{} ~{}, {}条,累计:{}条".format( _res[0]["datetime"], _res[-1]["datetime"], len(_res), _pos)) else: break if len(_datas) >= max_data_size: break if len(_datas) == 0: self.write_error(u"{}分笔成交数据获取为空") return True, _datas except Exception as ex: self.write_error( "exception in get_transaction_data:{},{},{}".format( symbol, str(ex), traceback.format_exc())) self.write_error(u"当前异常服务器信息:{}".format(self.best_ip)) self.write_log(u"重置连接") self.api = None self.connect(is_reconnect=True) return False, ret_datas def save_cache(self, cache_folder, cache_symbol, cache_date, data_list): """保存文件到缓存""" os.makedirs(cache_folder, exist_ok=True) if not os.path.exists(cache_folder): self.write_error("缓存目录不存在:{},不能保存".format(cache_folder)) return cache_folder_year_month = os.path.join(cache_folder, cache_date[:6]) os.makedirs(cache_folder_year_month, exist_ok=True) save_file = os.path.join(cache_folder_year_month, "{}_{}.pkz2".format(cache_symbol, cache_date)) try: with bz2.BZ2File(save_file, "wb") as f: pickle.dump(data_list, f) self.write_log(u"缓存成功:{}".format(save_file)) except Exception as ex: self.write_error(u"缓存写入异常:{}".format(str(ex))) def load_cache(self, cache_folder, cache_symbol, cache_date): """加载缓存数据""" if not os.path.exists(cache_folder): self.write_error("缓存目录:{}不存在,不能读取".format(cache_folder)) return None cache_folder_year_month = os.path.join(cache_folder, cache_date[:6]) if not os.path.exists(cache_folder_year_month): self.write_error("缓存目录:{}不存在,不能读取".format(cache_folder_year_month)) return None cache_file = os.path.join( cache_folder_year_month, "{}_{}.pkz2".format(cache_symbol, cache_date)) if not os.path.isfile(cache_file): self.write_error("缓存文件:{}不存在,不能读取".format(cache_file)) return None with bz2.BZ2File(cache_file, "rb") as f: data = pickle.load(f) return data return None def get_history_transaction_data(self, symbol, trading_date, cache_folder=None): """获取当某一交易日的历史成交记录""" ret_datas = [] # trading_date, 转换为数字类型得日期 if isinstance(trading_date, datetime): trading_date = trading_date.strftime("%Y%m%d") if isinstance(trading_date, str): trading_date = int(trading_date.replace("-", "")) self.connect() cache_symbol = symbol cache_date = str(trading_date) max_data_size = sys.maxsize symbol = symbol.upper() if "99" in symbol: # 查询的是指数合约 symbol = symbol.replace("99", "L9") tdx_index_symbol = symbol else: # 查询的是普通合约 tdx_index_symbol = get_underlying_symbol(symbol).upper() + "L9" q_size = QSIZE * 5 # 每秒 2个, 10小时 max_data_size = 1000000 # 优先从缓存加载 if cache_folder: buffer_data = self.load_cache(cache_folder, cache_symbol, cache_date) if buffer_data: self.write_log(u"使用缓存文件") return True, buffer_data self.write_log(u"开始下载{} 历史{}分笔数据".format(trading_date, symbol)) cur_trading_date = get_trading_date() if trading_date == int(cur_trading_date.replace("-", "")): return self.get_transaction_data(symbol) try: _datas = [] _pos = 0 while True: _res = self.api.get_history_transaction_data( market=self.symbol_market_dict.get(tdx_index_symbol, 0), date=trading_date, code=symbol, start=_pos, count=q_size) if _res is not None: for d in _res: dt = d.pop("date") # 星期1~星期6 if dt.hour >= 20 and 1 < dt.isoweekday() <= 6: dt = dt - timedelta(days=1) d.update({"datetime": dt}) elif dt.hour >= 20 and dt.isoweekday() == 1: # 星期一取得20点后数据 dt = dt - timedelta(days=3) d.update({"datetime": dt}) elif dt.hour < 8 and dt.isoweekday() == 1: # 星期一取得8点前数据 dt = dt - timedelta(days=3) d.update({"datetime": dt}) elif dt.hour >= 20 and dt.isoweekday() == 7: # 星期天取得20点后数据,肯定是星期五夜盘 dt = dt - timedelta(days=2) d.update({"datetime": dt}) elif dt.isoweekday() == 7: # 星期日取得其他时间,必然是 星期六凌晨的数据 dt = dt - timedelta(days=1) d.update({"datetime": dt}) else: d.update({"datetime": dt}) # 接口有bug,返回价格*1000,所以要除以1000 d.update({"price": d.get("price", 0) / 1000}) _datas = sorted(_res, key=lambda s: s["datetime"]) + _datas _pos += min(q_size, len(_res)) if _res is not None and len(_res) > 0: self.write_log(u"分段取分笔数据:{} ~{}, {}条,累计:{}条".format( _res[0]["datetime"], _res[-1]["datetime"], len(_res), _pos)) else: break if len(_datas) >= max_data_size: break if len(_datas) == 0: self.write_error(u"{}分笔成交数据获取为空".format(trading_date)) return False, _datas # 缓存文件 if cache_folder: self.save_cache(cache_folder, cache_symbol, cache_date, _datas) return True, _datas except Exception as ex: self.write_error( "exception in get_transaction_data:{},{},{}".format( symbol, str(ex), traceback.format_exc())) self.write_error(u"当前异常服务器信息:{}".format(self.best_ip)) self.write_log(u"重置连接") self.api = None self.connect(is_reconnect=True) return False, ret_datas def update_mi_contracts(self): # 连接通达信,获取主力合约 if not self.api: self.connect() mi_contract_quote_list = self.get_mi_contracts2() self.write_log(u"一共获取:{}个主力合约:{}".format( len(mi_contract_quote_list), [c.get("code") for c in mi_contract_quote_list])) should_save = False # 逐一更新主力合约数据 for mi_contract in mi_contract_quote_list: tdx_market_id = mi_contract.get("market") full_symbol = mi_contract.get("code") underlying_symbol = get_underlying_symbol(full_symbol).upper() if underlying_symbol in ["SC", "NR"]: vn_exchange = Exchange.INE else: vn_exchange = Tdx_Vn_Exchange_Map.get(str(tdx_market_id)) mi_symbol = get_real_symbol_by_exchange(full_symbol, vn_exchange) # 更新登记 短合约:真实主力合约 self.write_log("{},{},{},{},{}".format(tdx_market_id, full_symbol, underlying_symbol, mi_symbol, vn_exchange)) if underlying_symbol in self.future_contracts: info = self.future_contracts.get(underlying_symbol) if mi_symbol > info.get("mi_symbol"): self.write_log(u"主力合约变化:{} =>{}".format( info.get("mi_symbol"), mi_symbol)) info.update({ "mi_symbol": mi_symbol, "full_symbol": full_symbol }) self.future_contracts.update({underlying_symbol: info}) should_save = True else: # 添加到新合约中 # 这里缺少size和price_tick, margin_rate,当ctp_gateway启动时,会自动补充和修正完毕 info = { "underlying_symbol": underlying_symbol, "mi_symbol": mi_symbol, "full_symbol": full_symbol, "exchange": vn_exchange.value } self.write_log( u"新合约:{}, 需要待ctp连接后更新合约的size/price_tick/margin_rate". format(info)) self.future_contracts.update({underlying_symbol: info}) should_save = True if should_save: save_future_contracts(self.future_contracts)
num = api.get_instrument_count() all_contacts = sum([ api.get_instrument_info((int(num / 500) - i) * 500, 500) for i in range(int(num / 500) + 1) ], []) print(all_contacts) # 指数合约 index_contracts = api.get_instrument_quote_list(42, 3, 0, 100) # 主力合约 main_contracts = api.get_instrument_quote_list(60, 3, 0, 100) markets_code = api.get_markets() data = api.get_instrument_bars(TDXParams.KLINE_TYPE_DAILY, 8, "10000843", 0, 100) # 对所有合约处理,更新字典 指数合约-tdx市场,指数合约-交易所 for tdx_contract in all_contacts: tdx_symbol = tdx_contract.get('code', None) if tdx_symbol is None: continue tdx_market_id = tdx_contract.get('market') if str(tdx_market_id) in Tdx_Vn_Exchange_Map: TdxFutureData.symbol_exchange_dict.update( {tdx_symbol: Tdx_Vn_Exchange_Map.get(str(tdx_market_id))}) TdxFutureData.symbol_market_dict.update( {tdx_symbol: tdx_market_id}) # data = api.get_k_data(stock, '2015-01-01', '2020-01-14') # shsecdict = {} # shall = api.get_security_count(1)
import numpy as np from pytdx.config.hosts import hq_hosts api = TdxExHq_API(heartbeat=True) import time symol = "YL8" with api.connect('124.74.236.94', 7721): #[print (i) for i in api.get_markets()] while True: time.sleep(1) now = ("%s" % time.strftime('%Y.%m.%d %H:%M:%S', time.localtime(time.time()))) call = [] df = api.get_instrument_bars(category = TDXParams.KLINE_TYPE_1MIN, market = 29, code = symol, start = 0, count=200) df = pd.DataFrame(df) df = df[['datetime','open', 'high', 'low', 'close',]] print (df) df['ema_f_h'] = trend.ema(df.high, periods=34) df['ema_f_c'] = trend.ema(df.close, periods=34) df['ema_f_l'] = trend.ema(df.low, periods=34) df['ema_s_h'] = trend.ema(df.high, periods=144) df['ema_s_c'] = trend.ema(df.close, periods=144) df['ema_s_l'] = trend.ema(df.low, periods=144) df['ema_across'] = np.where(df.ema_f_c > df.ema_s_c, int(1), int(-1)) # 上穿1,下穿-1 # MACD(df, fast=55, slow=144, n=55) df["macd"] = trend.MACD(df.close, n_slow=144, n_fast=34, n_sign=34).macd()
class TdxFutureData(object): # ---------------------------------------------------------------------- def __init__(self, strategy, best_ip={}): """ 构造函数 :param strategy: 上层策略,主要用与使用write_log() """ self.api = None self.connection_status = False # 连接状态 self.best_ip = best_ip self.symbol_exchange_dict = {} # tdx合约与vn交易所的字典 self.symbol_market_dict = copy.copy( INIT_TDX_MARKET_MAP) # tdx合约与tdx市场的字典 self.strategy = strategy def write_log(self, content): if self.strategy: self.strategy.write_log(content) else: print(content) def write_error(self, content): if self.strategy: self.strategy.write_log(content, level=ERROR) else: print(content, file=sys.stderr) def connect(self, is_reconnect=False): """ 连接API :return: """ # 创建api连接对象实例 try: if self.api is None or not self.connection_status: self.write_log(u'开始连接通达信行情服务器') self.api = TdxExHq_API(heartbeat=True, auto_retry=True, raise_exception=True) # 选取最佳服务器 if is_reconnect or len(self.best_ip) == 0: self.best_ip = get_cache_ip() if len(self.best_ip) == 0: self.best_ip = self.select_best_ip() self.api.connect(self.best_ip['ip'], self.best_ip['port']) # 尝试获取市场合约统计 c = self.api.get_instrument_count() if c < 10: err_msg = u'该服务器IP {}/{}无响应'.format( self.best_ip['ip'], self.best_ip['port']) self.write_error(err_msg) else: self.write_log(u'创建tdx连接, IP: {}/{}'.format( self.best_ip['ip'], self.best_ip['port'])) # print(u'创建tdx连接, IP: {}/{}'.format(self.best_ip['ip'], self.best_ip['port'])) self.connection_status = True # if not is_reconnect: # 更新 symbol_exchange_dict , symbol_market_dict # self.qryInstrument() except Exception as ex: self.write_log(u'连接服务器tdx异常:{},{}'.format(str(ex), traceback.format_exc())) return # ---------------------------------------------------------------------- def ping(self, ip, port=7709): """ ping行情服务器 :param ip: :param port: :param type_: :return: """ apix = TdxExHq_API() __time1 = datetime.now() try: with apix.connect(ip, port): if apix.get_instrument_count() > 10000: _timestamp = datetime.now() - __time1 self.write_log(f'服务器{ip}:{port},耗时:{_timestamp}') return _timestamp else: self.write_log(f'该服务器IP {ip}无响应') return timedelta(9, 9, 0) except Exception: self.write_error(f'tdx ping服务器,异常的响应{ip}') return timedelta(9, 9, 0) # ---------------------------------------------------------------------- def select_best_ip(self): """ 选择行情服务器 :return: """ self.write_log(u'选择通达信行情服务器') data_future = [self.ping(x['ip'], x['port']) for x in TDX_FUTURE_HOSTS] best_future_ip = TDX_FUTURE_HOSTS[data_future.index(min(data_future))] self.write_log(u'选取 {}:{}'.format(best_future_ip['ip'], best_future_ip['port'])) # print(u'选取 {}:{}'.format(best_future_ip['ip'], best_future_ip['port'])) save_cache_ip(best_future_ip) return best_future_ip # ---------------------------------------------------------------------- def qryInstrument(self): """ 查询/更新合约信息 :return: """ # 取得所有的合约信息 num = self.api.get_instrument_count() if not isinstance(num, int): return all_contacts = sum([ self.api.get_instrument_info((int(num / 500) - i) * 500, 500) for i in range(int(num / 500) + 1) ], []) # [{"category":category,"market": int,"code":sting,"name":string,"desc":string},{}] # 对所有合约处理,更新字典 指数合约-tdx市场,指数合约-交易所 for tdx_contract in all_contacts: tdx_symbol = tdx_contract.get('code', None) if tdx_symbol is None: continue tdx_market_id = tdx_contract.get('market') if str(tdx_market_id) in Tdx_Vn_Exchange_Map: self.symbol_exchange_dict.update( {tdx_symbol: Tdx_Vn_Exchange_Map.get(str(tdx_market_id))}) self.symbol_market_dict.update({tdx_symbol: tdx_market_id}) # ---------------------------------------------------------------------- def get_bars(self, symbol, period, callback, bar_is_completed=False, bar_freq=1, start_dt=None): """ 返回k线数据 symbol:合约 period: 周期: 1min,3min,5min,15min,30min,1day,3day,1hour,2hour,4hour,6hour,12hour """ ret_bars = [] tdx_symbol = symbol.upper().replace('_', '') tdx_symbol = tdx_symbol.replace('99', 'L9') tdx_index_symbol = get_underlying_symbol(symbol) + 'L9' self.connect() if self.api is None: return False, ret_bars if period not in PERIOD_MAPPING.keys(): self.write_error(u'{} 周期{}不在下载清单中: {}'.format( datetime.now(), period, list(PERIOD_MAPPING.keys()))) return False, ret_bars # tdx_period = PERIOD_MAPPING.get(period) if start_dt is None: self.write_log(u'没有设置开始时间,缺省为10天前') qry_start_date = datetime.now() - timedelta(days=10) else: qry_start_date = start_dt end_date = datetime.combine(datetime.now() + timedelta(days=1), time(ALL_MARKET_END_HOUR, 0)) if qry_start_date > end_date: qry_start_date = end_date self.write_log('{}开始下载tdx:{} {}数据, {} to {}.'.format( datetime.now(), tdx_symbol, period, qry_start_date, end_date)) # print('{}开始下载tdx:{} {}数据, {} to {}.'.format(datetime.now(), tdx_symbol, tdx_period, last_date, end_date)) try: _start_date = end_date _bars = [] _pos = 0 while _start_date > qry_start_date: _res = self.api.get_instrument_bars( PERIOD_MAPPING[period], self.symbol_market_dict.get(tdx_index_symbol, 0), tdx_symbol, _pos, QSIZE) if _res is not None: _bars = _res + _bars _pos += QSIZE if _res is not None and len(_res) > 0: _start_date = _res[0]['datetime'] _start_date = datetime.strptime(_start_date, '%Y-%m-%d %H:%M') self.write_log(u'分段取数据开始:{}'.format(_start_date)) else: break if len(_bars) == 0: self.write_error('{} Handling {}, len1={}..., continue'.format( str(datetime.now()), tdx_symbol, len(_bars))) return False, ret_bars current_datetime = datetime.now() data = self.api.to_df(_bars) data = data.assign(datetime=to_datetime(data['datetime'])) data = data.assign(ticker=symbol) data['instrument_id'] = data['ticker'] # if future['market'] == 28 or future['market'] == 47: # # 大写字母: 郑州商品 or 中金所期货 # data['instrument_id'] = data['ticker'] # else: # data['instrument_id'] = data['ticker'].apply(lambda x: x.lower()) data['symbol'] = symbol data = data.drop([ 'year', 'month', 'day', 'hour', 'minute', 'price', 'amount', 'ticker' ], errors='ignore', axis=1) data = data.rename(index=str, columns={ 'position': 'open_interest', 'trade': 'volume', }) if len(data) == 0: print('{} Handling {}, len2={}..., continue'.format( str(datetime.now()), tdx_symbol, len(data))) return False, ret_bars data['total_turnover'] = data['volume'] data["limit_down"] = 0 data["limit_up"] = 999999 data['trading_date'] = data['datetime'] data['trading_date'] = data['trading_date'].apply( lambda x: (x.strftime('%Y-%m-%d'))) monday_ts = data['datetime'].dt.weekday == 0 # 星期一 night_ts1 = data['datetime'].dt.hour > ALL_MARKET_END_HOUR night_ts2 = data['datetime'].dt.hour < ALL_MARKET_BEGIN_HOUR data.loc[night_ts1, 'datetime'] -= timedelta( days=1) # 所有日期的夜盘(21:00~24:00), 减一天 monday_ts1 = monday_ts & night_ts1 # 星期一的夜盘(21:00~24:00), 再减两天 data.loc[monday_ts1, 'datetime'] -= timedelta(days=2) monday_ts2 = monday_ts & night_ts2 # 星期一的夜盘(00:00~04:00), 再减两天 data.loc[monday_ts2, 'datetime'] -= timedelta(days=2) # data['datetime'] -= timedelta(minutes=1) # 直接给Strategy使用, RiceQuant格式, 不需要减1分钟 data['dt_datetime'] = data['datetime'] data['date'] = data['datetime'].apply(lambda x: (x.strftime('%Y-%m-%d'))) data['time'] = data['datetime'].apply(lambda x: (x.strftime('%H:%M:%S'))) data['datetime'] = data['datetime'].apply( lambda x: float(x.strftime('%Y%m%d%H%M%S'))) data = data.set_index('dt_datetime', drop=False) # data = data[int(last_date.strftime('%Y%m%d%H%M%S')):int(end_date.strftime('%Y%m%d%H%M%S'))] # data = data[str(last_date):str(end_date)] for index, row in data.iterrows(): add_bar = BarData() try: add_bar.symbol = row['symbol'] add_bar.datetime = index add_bar.date = row['date'] add_bar.time = row['time'] add_bar.trading_date = row['trading_date'] add_bar.open = float(row['open']) add_bar.high = float(row['high']) add_bar.low = float(row['low']) add_bar.close = float(row['close']) add_bar.volume = float(row['volume']) add_bar.openInterest = float(row['open_interest']) except Exception as ex: self.write_error( 'error when convert bar:{},ex:{},t:{}'.format( row, str(ex), traceback.format_exc())) # print('error when convert bar:{},ex:{},t:{}'.format(row, str(ex), traceback.format_exc())) return False if start_dt is not None and index < start_dt: continue ret_bars.append(add_bar) if callback is not None: freq = bar_freq bar_is_completed = True if period != '1min' and index == data['dt_datetime'][-1]: # 最后一个bar,可能是不完整的,强制修改 # - 5min修改后freq基本正确 # - 1day在VNPY合成时不关心已经收到多少Bar, 所以影响也不大 # - 但其它分钟周期因为不好精确到每个品种, 修改后的freq可能有错 if index > current_datetime: bar_is_completed = False # 根据秒数算的话,要+1,例如13:31,freq=31,第31根bar freq = NUM_MINUTE_MAPPING[period] - int( (index - current_datetime).total_seconds() / 60) callback(add_bar, bar_is_completed, freq) return True, ret_bars except Exception as ex: self.write_error('exception in get:{},{},{}'.format( tdx_symbol, str(ex), traceback.format_exc())) # print('exception in get:{},{},{}'.format(tdx_symbol,str(ex), traceback.format_exc())) self.write_log(u'重置连接') self.api = None self.connect(is_reconnect=True) return False, ret_bars def get_price(self, symbol): """获取最新价格""" tdx_symbol = symbol.upper().replace('_', '') short_symbol = get_underlying_symbol(tdx_symbol).upper() if tdx_symbol.endswith('99'): query_symbol = tdx_symbol.replace('99', 'L9') else: query_symbol = get_full_symbol(tdx_symbol) if query_symbol != tdx_symbol: self.write_log('转换合约:{}=>{}'.format(tdx_symbol, query_symbol)) tdx_index_symbol = short_symbol + 'L9' self.connect() if self.api is None: return 0 market_id = self.symbol_market_dict.get(tdx_index_symbol, 0) _res = self.api.get_instrument_quote(market_id, query_symbol) if not isinstance(_res, list): return 0 if len(_res) == 0: return 0 return float(_res[0].get('price', 0)) def get_99_contracts(self): """ 获取指数合约 :return: dict list """ self.connect() result = self.api.get_instrument_quote_list(42, 3, 0, 100) return result def get_mi_contracts(self): """ 获取主力合约 :return: dict list """ self.connect() result = self.api.get_instrument_quote_list(60, 3, 0, 100) return result def get_contracts(self, exchange): self.connect() market_id = Vn_Tdx_Exchange_Map.get(exchange, None) if market_id is None: print(u'市场:{}配置不在Vn_Tdx_Exchange_Map:{}中,不能取市场下所有合约'.format( exchange, Vn_Tdx_Exchange_Map)) return [] index = 0 count = 100 results = [] while (True): print(u'查询{}下:{}~{}个合约'.format(exchange, index, index + count)) result = self.api.get_instrument_quote_list( int(market_id), 3, index, count) results.extend(result) index += count if len(result) < count: break return results def get_mi_contracts2(self): """ 获取主力合约""" self.connect() contracts = [] for exchange in Vn_Tdx_Exchange_Map.keys(): contracts.extend(self.get_mi_contracts_from_exchange(exchange)) return contracts def get_mi_contracts_from_exchange(self, exchange): contracts = self.get_contracts(exchange) if len(contracts) == 0: print(u'异常,未能获取{}下合约信息'.format(exchange)) return [] mi_contracts = [] short_contract_dict = {} for contract in contracts: # 排除指数合约 code = contract.get('code') if code[-2:] in ['L9', 'L8', 'L0', 'L1', 'L2', 'L3', '50'] or\ (exchange == Exchange.CFFEX and code[-3:] in ['300', '500']): continue short_symbol = get_underlying_symbol(code).upper() contract_list = short_contract_dict.get(short_symbol, []) contract_list.append(contract) short_contract_dict.update({short_symbol: contract_list}) for k, v in short_contract_dict.items(): sorted_list = sorted(v, key=lambda c: c['ZongLiang']) mi_contracts.append(sorted_list[-1]) return mi_contracts def get_markets(self): """ 获取市场代码 :return: """ self.connect() result = self.api.get_markets() return result def get_transaction_data(self, symbol): """获取当前交易日的历史成交记录""" ret_datas = [] max_data_size = sys.maxsize symbol = symbol.upper() if '99' in symbol: # 查询的是指数合约 symbol = symbol.replace('99', 'L9') tdx_index_symbol = symbol else: # 查询的是普通合约 tdx_index_symbol = get_underlying_symbol(symbol).upper() + 'L9' self.connect() q_size = QSIZE * 5 # 每秒 2个, 10小时 max_data_size = 1000000 self.write_log(u'开始下载{}当日分笔数据'.format(symbol)) try: _datas = [] _pos = 0 while (True): _res = self.api.get_transaction_data( market=self.symbol_market_dict.get(tdx_index_symbol, 0), code=symbol, start=_pos, count=q_size) if _res is not None: for d in _res: dt = d.pop('date') # 星期1~星期6 if dt.hour >= 20 and 1 < dt.isoweekday() <= 6: dt = dt - timedelta(days=1) elif dt.hour >= 20 and dt.isoweekday() == 1: # 星期一取得20点后数据 dt = dt - timedelta(days=3) elif dt.hour < 8 and dt.isoweekday() == 1: # 星期一取得8点前数据 dt = dt - timedelta(days=3) elif dt.hour >= 20 and dt.isoweekday() == 7: # 星期天取得20点后数据,肯定是星期五夜盘 dt = dt - timedelta(days=2) elif dt.isoweekday() == 7: # 星期日取得其他时间,必然是 星期六凌晨的数据 dt = dt - timedelta(days=1) d.update({'datetime': dt}) # 接口有bug,返回价格*1000,所以要除以1000 d.update({'price': d.get('price', 0) / 1000}) _datas = sorted(_res, key=lambda s: s['datetime']) + _datas _pos += min(q_size, len(_res)) if _res is not None and len(_res) > 0: self.write_log(u'分段取分笔数据:{} ~{}, {}条,累计:{}条'.format( _res[0]['datetime'], _res[-1]['datetime'], len(_res), _pos)) else: break if len(_datas) >= max_data_size: break if len(_datas) == 0: self.write_error(u'{}分笔成交数据获取为空') return True, _datas except Exception as ex: self.write_error( 'exception in get_transaction_data:{},{},{}'.format( symbol, str(ex), traceback.format_exc())) self.write_error(u'当前异常服务器信息:{}'.format(self.best_ip)) self.write_log(u'重置连接') self.api = None self.connect(is_reconnect=True) return False, ret_datas def save_cache(self, cache_folder, cache_symbol, cache_date, data_list): """保存文件到缓存""" os.makedirs(cache_folder, exist_ok=True) if not os.path.exists(cache_folder): self.write_error('缓存目录不存在:{},不能保存'.format(cache_folder)) return cache_folder_year_month = os.path.join(cache_folder, cache_date[:6]) os.makedirs(cache_folder_year_month, exist_ok=True) save_file = os.path.join(cache_folder_year_month, '{}_{}.pkz2'.format(cache_symbol, cache_date)) try: with bz2.BZ2File(save_file, 'wb') as f: pickle.dump(data_list, f) self.write_log(u'缓存成功:{}'.format(save_file)) except Exception as ex: self.write_error(u'缓存写入异常:{}'.format(str(ex))) def load_cache(self, cache_folder, cache_symbol, cache_date): """加载缓存数据""" if not os.path.exists(cache_folder): self.write_error('缓存目录:{}不存在,不能读取'.format(cache_folder)) return None cache_folder_year_month = os.path.join(cache_folder, cache_date[:6]) if not os.path.exists(cache_folder_year_month): self.write_error('缓存目录:{}不存在,不能读取'.format(cache_folder_year_month)) return None cache_file = os.path.join( cache_folder_year_month, '{}_{}.pkz2'.format(cache_symbol, cache_date)) if not os.path.isfile(cache_file): self.write_error('缓存文件:{}不存在,不能读取'.format(cache_file)) return None with bz2.BZ2File(cache_file, 'rb') as f: data = pickle.load(f) return data return None def get_history_transaction_data(self, symbol, date, cache_folder=None): """获取当某一交易日的历史成交记录""" ret_datas = [] if isinstance(date, datetime): date = date.strftime('%Y%m%d') if isinstance(date, str): date = int(date) self.connect() cache_symbol = symbol cache_date = str(date) max_data_size = sys.maxsize symbol = symbol.upper() if '99' in symbol: # 查询的是指数合约 symbol = symbol.replace('99', 'L9') tdx_index_symbol = symbol else: # 查询的是普通合约 tdx_index_symbol = get_underlying_symbol(symbol).upper() + 'L9' q_size = QSIZE * 5 # 每秒 2个, 10小时 max_data_size = 1000000 # 优先从缓存加载 if cache_folder: buffer_data = self.load_cache(cache_folder, cache_symbol, cache_date) if buffer_data: self.write_log(u'使用缓存文件') return True, buffer_data self.write_log(u'开始下载{} 历史{}分笔数据'.format(date, symbol)) cur_trading_date = get_trading_date() if date == int(cur_trading_date.replace('-', '')): return self.get_transaction_data(symbol) try: _datas = [] _pos = 0 while (True): _res = self.api.get_history_transaction_data( market=self.symbol_market_dict.get(tdx_index_symbol, 0), date=date, code=symbol, start=_pos, count=q_size) if _res is not None: for d in _res: dt = d.pop('date') # 星期1~星期6 if dt.hour >= 20 and 1 < dt.isoweekday() <= 6: dt = dt - timedelta(days=1) d.update({'datetime': dt}) elif dt.hour >= 20 and dt.isoweekday() == 1: # 星期一取得20点后数据 dt = dt - timedelta(days=3) d.update({'datetime': dt}) elif dt.hour < 8 and dt.isoweekday() == 1: # 星期一取得8点前数据 dt = dt - timedelta(days=3) d.update({'datetime': dt}) elif dt.hour >= 20 and dt.isoweekday() == 7: # 星期天取得20点后数据,肯定是星期五夜盘 dt = dt - timedelta(days=2) d.update({'datetime': dt}) elif dt.isoweekday() == 7: # 星期日取得其他时间,必然是 星期六凌晨的数据 dt = dt - timedelta(days=1) d.update({'datetime': dt}) else: d.update({'datetime': dt}) # 接口有bug,返回价格*1000,所以要除以1000 d.update({'price': d.get('price', 0) / 1000}) _datas = sorted(_res, key=lambda s: s['datetime']) + _datas _pos += min(q_size, len(_res)) if _res is not None and len(_res) > 0: self.write_log(u'分段取分笔数据:{} ~{}, {}条,累计:{}条'.format( _res[0]['datetime'], _res[-1]['datetime'], len(_res), _pos)) else: break if len(_datas) >= max_data_size: break if len(_datas) == 0: self.write_error(u'{}分笔成交数据获取为空'.format(date)) return False, _datas # 缓存文件 if cache_folder: self.save_cache(cache_folder, cache_symbol, cache_date, _datas) return True, _datas except Exception as ex: self.write_error( 'exception in get_transaction_data:{},{},{}'.format( symbol, str(ex), traceback.format_exc())) self.write_error(u'当前异常服务器信息:{}'.format(self.best_ip)) self.write_log(u'重置连接') self.api = None self.connect(is_reconnect=True) return False, ret_datas
("low", low), ("close", close), ("position", position), ("trade", trade), ("price", price), ("year", year), ("month", month), ("day", day), ("hour", hour), ("minute", minute), ("datetime", "%d-%02d-%02d %02d:%02d" % (year, month, day, hour, minute)), ("amount", amount), ]) klines.append(kline) return klines if __name__ == '__main__': from pytdx.exhq import TdxExHq_API from pytdx.params import TDXParams api = TdxExHq_API() # cmd = GetInstrumentBars(api) # cmd.setParams(4, 7, "10000843", 0, 10) # print(cmd.send_pkg) with api.connect('61.152.107.141', 7727): print(api.to_df(api.get_instrument_bars(TDXParams.KLINE_TYPE_EXHQ_1MIN, 74, 'BABA')).tail()) print(api.to_df(api.get_instrument_bars(TDXParams.KLINE_TYPE_DAILY, 31, '00001')).tail())
self.category, body_buf, pos) (open_price, high, low, close, position, trade, price) = struct.unpack("<ffffIIf", body_buf[pos:pos + 28]) pos += 28 kline = OrderedDict([ ("open", open_price), ("high", high), ("low", low), ("close", close), ("position", position), ("trade", trade), ("price", price), ("year", year), ("month", month), ("day", day), ("hour", hour), ("minute", minute), ("datetime", "%d-%02d-%02d %02d:%02d" % (year, month, day, hour, minute)) ]) klines.append(kline) return klines if __name__ == '__main__': from pytdx.exhq import TdxExHq_API from pytdx.params import TDXParams api = TdxExHq_API() # cmd = GetInstrumentBars(api) # cmd.setParams(4, 7, "10000843", 0, 10) # print(cmd.send_pkg) with api.connect('61.152.107.141', 7727): # print(api.to_df(api.get_instrument_bars(TDXParams.KLINE_TYPE_EXHQ_1MIN, 74, 'BABA')).tail()) print( api.to_df( api.get_instrument_bars(TDXParams.KLINE_TYPE_DAILY, 29, 'C1801')))
class Datacent: def __init__(self): self.cf = config self.tools = tdx_tools self.qihuo_api = TdxExHq_API() self.gupiao_api = TdxHq_API() self.cflen = 0 self.table = pt.PrettyTable() def gupiao_connectSer(self): self.gupiao_api.connect('58.63.254.191', 7709) gupiaoret = self.gupiao_api.connect('58.63.254.191', 7709) if gupiaoret == False: print("股票没有连接。。。") return gupiaoret else: print("已连接股票数据服务") gupiaoret = True return gupiaoret def qihuo_connectSer(self): self.qihuo_api.connect('124.74.236.94', 7721) qihuoret = self.qihuo_api.connect('124.74.236.94', 7721) if qihuoret == False: print("期货没有连接。。。") return qihuoret else: print("已连接期货数据服务") qihuoret = True return qihuoret def realtimestock(self, ): # stlist提取多只个股的信息 stokname是把个股的中文名再放进去 stlist = [] stname = [] for i in config.cfstock: st = (i["marketid"], i["code"]) stlist.append(st) stname.append(i["stockname"]) rtsdata = self.gupiao_api.get_security_quotes(stlist) rtsdata = pd.DataFrame(rtsdata) # print (rtsdata) # print (rtsvol) if len(rtsdata) >= 0: rtsdata['zd'] = rtsdata.price - rtsdata.last_close rtsdata['zdf'] = (rtsdata.price - rtsdata.last_close) / rtsdata.last_close rtsdata['zdf'] = rtsdata.zdf.map(lambda x: format(x, '.2%')) rtsdata['stname'] = stname rtsdata = rtsdata[[ 'stname', 'code', 'price', 'last_close', 'ask1', 'bid1', 'ask_vol1', 'bid_vol1', 'zd', 'zdf' ]] rtsdata = rtsdata.round(2) # rtsvol = realtimevol() # print (rtsvol) # rtsdata = pd.concat([rtsdata,rtsvol],axis=1) return rtsdata else: return np.nan def candlelink(self): df['看涨抱线'] = np.where((df.open_last_1 > df.close_last_1) & (df.open < df.close_last_1) & (df.close > df.open_last_1), 1, 0) df['看涨孕线'] = np.where((df.open_last_1 > df.close_last_1) & (df.open < df.close) & (np.abs(df.high - df.low) < np.abs(df.open_last_1 - df.close_last_1)) & (df.open > df.close_last_1), 1, 0) df['看跌吞没'] = np.where((df.open_last_1 < df.close_last_1) & (df.open > df.close) & (df.close < df.open_last_1) & (df.open > df.close_last_1), 1, 0) df['看跌孕线'] = np.where((df.open_last_1 < df.close_last_1) & (df.open > df.close) & (np.abs(df.high - df.low) < np.abs(df.open_last_1 - df.close_last_1)) & (df.open < df.close_last_1), 1, 0) df['看涨刺透'] = np.where( (df.open_last_1 > df.close_last_1) & (df.open < df.close) & (df.close > (np.abs(df.close_last_1 - df.open_last_1) / 2) + df.close_last_1) & (df.open < df.close) & (df.open_last_1 > df.close) & (df.open_last_1 > df.open), 1, 0) df['乌云盖顶'] = np.where( (df.open_last_1 < df.close_last_1) & (df.open > df.close) & (df.close < (np.abs(df.close_last_1 - df.open_last_1) / 2) + df.open_last_1) & (df.close_last_1 < df.open) & (df.open_last_1 < df.close), 1, 0) return df def qihuoK( self, cflen, ): #cflen = 0 data = self.qihuo_api.get_instrument_bars( self.cf.category, int(self.cf.cfqihuo[cflen]["marketid"]), self.cf.cfqihuo[cflen]['code'], 0, self.cf.categorycount) #7: 扩展行情查询k线数据 #sh = self.api.to_df(self.api.get_index_bars(3,1,"000001",1,400)) #获得指数 df = pd.DataFrame( data, columns=['datetime', 'open', 'high', 'low', 'close'], ) df['stockname'] = self.cf.cfqihuo[cflen]["stockname"] df['code'] = self.cf.cfqihuo[cflen]["code"] df['dc_hband'] = self.tools.donchian_channel_hband( df.close, self.cf.dc_count) df['dc_lband'] = self.tools.donchian_channel_lband( df.close, self.cf.dc_count) df['dc_break'] = np.where( (df['close'] > df['dc_hband'].shift(1)), 1, np.where((df['close'] < df['dc_lband'].shift(1)), -1, 0)) #这里是算突破的信号,当往上突破为1,当往下突破为-1,震荡区间为0 #df = df.set_index('code') df["sma5"] = df['close'].rolling(5).mean() df["sma10"] = df['close'].rolling(10).mean() df["sma20"] = df['close'].rolling(20).mean() df["sma89"] = df['close'].rolling(89).mean() df["sma144"] = df['close'].rolling(144).mean() df["sma89144"] = np.where(df['close'] > df['sma89'], 1, np.where(df['close'] < df['sma144'], -1, 0)) df["sma51020"] = np.where(df['close'] > df['sma5'], 1, np.where(df['close'] < df['sma20'], -1, 0)) df['open_last_1'] = df['open'].shift(1) df['high_last_1'] = df['high'].shift(1) df["low_last_1"] = df['low'].shift(1) df['close_last_1'] = df['close'].shift(1) df["rsi"] = ta.momentum.rsi( df.close, n=14, ) df['stoch'] = ta.momentum.stoch(df.high, df.low, df.close, 14) df['看涨抱线'] = np.where((df.open_last_1 > df.close_last_1) & (df.open < df.close_last_1) & (df.close > df.open_last_1), 1, 0) df['看涨孕线'] = np.where((df.open_last_1 > df.close_last_1) & (df.open < df.close) & (np.abs(df.high - df.low) < np.abs(df.open_last_1 - df.close_last_1)) & (df.open > df.close_last_1), 1, 0) df['看跌吞没'] = np.where((df.open_last_1 < df.close_last_1) & (df.open > df.close) & (df.close < df.open_last_1) & (df.open > df.close_last_1), 1, 0) df['看跌孕线'] = np.where((df.open_last_1 < df.close_last_1) & (df.open > df.close) & (np.abs(df.high - df.low) < np.abs(df.open_last_1 - df.close_last_1)) & (df.open < df.close_last_1), 1, 0) df['看涨刺透'] = np.where( (df.open_last_1 > df.close_last_1) & (df.open < df.close) & (df.close > (np.abs(df.close_last_1 - df.open_last_1) / 2) + df.close_last_1) & (df.open < df.close) & (df.open_last_1 > df.close) & (df.open_last_1 > df.open), 1, 0) df['乌云盖顶'] = np.where( (df.open_last_1 < df.close_last_1) & (df.open > df.close) & (df.close < (np.abs(df.close_last_1 - df.open_last_1) / 2) + df.open_last_1) & (df.close_last_1 < df.open) & (df.open_last_1 < df.close), 1, 0) df['看涨一线穿'] = np.where( (df.sma5 > df.sma10) & (df.open < df.sma10) & (df.close > df.sma5) & (df.close > df.close_last_1), 1, 0) df['看跌一线穿'] = np.where( (df.sma5 < df.sma10) & (df.open > df.sma10) & (df.close < df.sma5) & (df.close < df.close_last_1), 1, 0) df = df.dropna() df = self.tools.SuperTrend( df, period=self.cf.st_period_fast, multiplier=self.cf.st_mult_fast, ) # print (stockdf.T) # if len(stockdf) >0: # skout.append(stockdf.iloc[-1]) # else: # continue; # print (stockdf) stockdf = pd.DataFrame(df).reset_index().drop( columns=['index']) #将个股最后一条的数据整合到一个新的df 里并重置列名 stockdf = stockdf.round(2) return stockdf def gupiaoK( self, cflen, ): #category 从confg文件中导入配置 修改获取K线的周期 ''' sh = api.to_df(api.get_index_bars(0,0,"399001",1,400)) #获得指数 return sh ''' # skout = [] # for i in self.cf.cfstock: data = self.gupiao_api.get_security_bars( self.cf.category, int(self.cf.cfstock[cflen]["marketid"]), self.cf.cfstock[cflen]['code'], 0, self.cf.categorycount) df = pd.DataFrame( data, columns=['datetime', 'open', 'high', 'low', 'close'], ) df['stockname'] = self.cf.cfstock[cflen]["stockname"] df['code'] = self.cf.cfstock[cflen]["code"] df['dc_hband'] = self.tools.donchian_channel_hband( df.close, self.cf.dc_count) df['dc_lband'] = self.tools.donchian_channel_lband( df.close, self.cf.dc_count) df['dc_break'] = np.where( (df['close'] > df['dc_hband'].shift(1)), 1, np.where((df['close'] < df['dc_lband'].shift(1)), -1, 0)) #这里是算突破的信号,当往上突破为1,当往下突破为-1,震荡区间为0 #df = df.set_index('code') df["sma5"] = df['close'].rolling(5).mean() df["sma10"] = df['close'].rolling(10).mean() df["sma20"] = df['close'].rolling(20).mean() df["sma89"] = df['close'].rolling(89).mean() df["sma144"] = df['close'].rolling(144).mean() df["sma89144"] = np.where(df['close'] > df['sma89'], 1, np.where(df['close'] < df['sma144'], -1, 0)) df["sma51020"] = np.where(df['close'] > df['sma5'], 1, np.where(df['close'] < df['sma20'], -1, 0)) df['open_last_1'] = df['open'].shift(1) df['high_last_1'] = df['high'].shift(1) df["low_last_1"] = df['low'].shift(1) df['close_last_1'] = df['close'].shift(1) df['看涨抱线'] = np.where((df.open_last_1 > df.close_last_1) & (df.open < df.close_last_1) & (df.close > df.open_last_1), 1, 0) df['看涨孕线'] = np.where((df.open_last_1 > df.close_last_1) & (df.open < df.close) & (np.abs(df.high - df.low) < np.abs(df.open_last_1 - df.close_last_1)) & (df.open > df.close_last_1), 1, 0) df['看跌吞没'] = np.where((df.open_last_1 < df.close_last_1) & (df.open > df.close) & (df.close < df.open_last_1) & (df.open > df.close_last_1), 1, 0) df['看跌孕线'] = np.where((df.open_last_1 < df.close_last_1) & (df.open > df.close) & (np.abs(df.high - df.low) < np.abs(df.open_last_1 - df.close_last_1)) & (df.open < df.close_last_1), 1, 0) df['看涨刺透'] = np.where( (df.open_last_1 > df.close_last_1) & (df.open < df.close) & (df.close > (np.abs(df.close_last_1 - df.open_last_1) / 2) + df.close_last_1) & (df.open < df.close) & (df.open_last_1 > df.close) & (df.open_last_1 > df.open), 1, 0) df['乌云盖顶'] = np.where( (df.open_last_1 < df.close_last_1) & (df.open > df.close) & (df.close < (np.abs(df.close_last_1 - df.open_last_1) / 2) + df.open_last_1) & (df.close_last_1 < df.open) & (df.open_last_1 < df.close), 1, 0) df['看涨一线穿'] = np.where( (df.sma5 > df.sma10) & (df.open < df.sma10) & (df.close > df.sma5) & (df.close > df.close_last_1), 1, 0) df['看跌一线穿'] = np.where( (df.sma5 < df.sma10) & (df.open > df.sma10) & (df.close < df.sma5) & (df.close < df.close_last_1), 1, 0) df = df.dropna() df = self.tools.SuperTrend( df, period=self.cf.st_period_fast, multiplier=self.cf.st_mult_fast, ) # print (stockdf.T) # if len(stockdf) >0: # skout.append(stockdf.iloc[-1]) # else: # continue; # print (stockdf) stockdf = pd.DataFrame(df).reset_index().drop( columns=['index']) #将个股最后一条的数据整合到一个新的df 里并重置列名 stockdf = stockdf.round(2) return stockdf
def test_all_functions(): symbol_params = [[47, "IF1709"], [8, "10000889"], [31, "00020"], [47, "IFL0"], [31, "00700"]] api = TdxExHq_API(auto_retry=True) try: with api.connect('121.14.110.210', 7727, time_out=30): log.info("获取市场代码") data = api.get_markets() assert data is not None assert type(data) is list assert len(data) > 0 log.info("查询市场中商品数量") data = api.get_instrument_count() assert data is not None assert data > 0 log.info("查询五档行情") for params in symbol_params: data = api.get_instrument_quote(*params) print(data) assert data is not None assert type(data) is list assert len(data) > 0 # log.info("查询分时行情") for params in symbol_params: data = api.get_minute_time_data(*params) assert data is not None assert type(data) is list assert len(data) >= 0 log.info("查询历史分时行情") for params in symbol_params: data = api.get_history_minute_time_data( params[0], params[1], 20170811) assert data is not None assert type(data) is list assert len(data) >= 0 log.info("查询分时成交") for params in symbol_params: data = api.get_transaction_data(*params) assert data is not None assert type(data) is list assert len(data) >= 0 log.info("查询历史分时成交") for params in symbol_params: data = api.get_history_transaction_data( params[0], params[1], 20170811) assert data is not None assert type(data) is list assert len(data) >= 0 log.info("查询k线") for params in symbol_params: data = api.get_instrument_bars(TDXParams.KLINE_TYPE_DAILY, params[0], params[1]) assert data is not None assert type(data) is list assert len(data) >= 0 log.info("查询代码列表") data = api.get_instrument_info(10000, 98) assert data is not None assert type(data) is list assert len(data) > 0 except socket.timeout as e: pass
class data(object): def __init__(self, heartbeat=True, isstock=False): self.api = None self.isstock = isstock self.heartbeat = heartbeat self.number = 80000 self.market = None self.datatype = 0 self.result = [] self.TDX_IP_SETS = [ "119.147.86.168", '119.97.185.5' '202.103.36.71', '139,196,185,253', '61.152.107.171', ] # 218.75.74.103:7721 60.12.15.21:7721 self.TDX_IP_SETS_STOCK = [ '119.147.164.60', '218.75.126.9', '115.238.90.165', '124.160.88.183', '60.12.136.250', '218.108.98.244', '218.108.47.69', '14.17.75.71', '180.153.39.51' ] self.file_incon = FILE_INCON self.file_tdxhy = FILE_TDXHY self.file_tdxzs = FILE_TDXZS self.weight = {} def _get_incon(self, ): '''获取行业分类代码 ''' f = open(self.file_incon, "rb") data = f.read() strings = data.decode("gbk", 'ignore').rstrip("\x00").replace("\r\n", "\n") data = strings.split("######") rst = {} for hystr in data: key = re.findall(r'#.*', hystr) if key == ['#TDXNHY']: hylst = hystr.replace("#TDXNHY", "").strip("\n").split("\n") for item in hylst: k, v = item.split("|") rst[k] = [v] return rst def _get_tdxhy(self, islocal=True): '''获取股票和行业对应列表 ''' if islocal: stocklist = HY_WEIGHT.keys() else: stocklist = list(ts.get_stock_basics().index) #获取全市场股票代码 rst = self._get_incon() f = open(self.file_tdxhy, "rb") data = f.read().decode("gbk", 'ignore').rstrip("\x00").replace( "\r\n", "\n").strip("\n").split("\n") for i in data: _, code, tdxhy, _, _ = i.split("|") if tdxhy != "T00" and code in stocklist: rst[tdxhy].append(code) return rst def _get_tdxzs(self, islocal=True): '''生成通达性版块代码对应股票列表 ''' dct = {} rst = self._get_tdxhy(islocal=islocal) f = open(self.file_tdxzs, "rb") data = f.read().decode("gbk", 'ignore').rstrip("\x00").replace( "\r\n", "\n").strip("\n").split("\n") for i in data: name, code, _, _, _, hy = i.split("|") code = int(code) if 880301 <= code and 880497 >= code and hy in rst.keys(): k = hy[:5] if not dct.__contains__(k): dct[k] = {"name": "", "code": "", "stocklist": []} if k == hy: dct[k]["name"] = name dct[k]["code"] = code dct[k]["stocklist"].extend(rst[hy][1:]) return dct def get_tdxhy_list(self, islocal=True): '''获取通达信行业板块指数对应的股票列表 ''' return self._get_tdxzs(islocal) def get_weight(self, htlist={}, islocal=True): '''获取行业板块个股权重,流动市值为权重系数 备注:回测是为方便处理,以最后一天的权重系数作为历史上的权重 ''' if islocal: self.weight = HY_WEIGHT else: if not htlist: htlist = self.get_tdxhy_list(islocal) tasks = [] for v in htlist.values(): tasks.append(gevent.spawn(self.get_latest_ltsz, v["stocklist"])) gevent.joinall(tasks) return self.weight def get_latest_ltsz(self, stocks=[]): '''获取最新流通市值,千万为单位,取整 ''' unit = 10000000 for code in stocks: mk = self._select_market_code(code) print(mk, code) try: ltgb = self.api.get_finance_info(mk, code)["liutongguben"] price = self.api.get_security_bars(4, mk, code, 0, 1)[0]["close"] ltsz = int(ltgb * price / unit) self.weight[code] = ltsz except: print("*****", code) return self.weight @property def mdb(self): '''设置数据库连接 ''' if not hasattr(self, "_db"): self._db = MongoDB() return self._db @property def starttime(self): '''采样起始时间 ''' if not hasattr(self, "_starttime"): self._starttime = "2006-01-01" return self._starttime @starttime.setter def starttime(self, value): self._starttime = value return self._starttime @property def endtime(self): '''采样结束时间 ''' if not hasattr(self, "_endtime"): self._endtime = str(datetime.date.today()) return self._endtime @endtime.setter def endtime(self, value): self._endtime = value return self._endtime def getdata_m( self, collection, db=MIN_STOCK_DB, project={ "open": 1, "high": 1, "low": 1, "close": 1, "datetime": 1, "vol": 1, "_id": 0 }): '''从数据库获取分钟数据 ''' filt = {"datetime": {"$gte": self.starttime, "$lte": self.endtime}} data = [ i for i in self.mdb._dbclient(db)[collection].find(filt, project) ] return pd.DataFrame(data) def getcollections(self, db=MIN_STOCK_DB): data = self.mdb.getallcollections(db) return data def clean_m(self, df, field="datetime"): '''数据库数据有些格式有问题,比如,有些11.30有数据,有些没有。有些13.00有数据,有些没有 导致用分钟数据生成30分钟数据时出现异常 ''' df[field] = df[field].str.replace("13:00", "11:30") df.set_index(field, inplace=True, drop=False) df.index = pd.DatetimeIndex(df.index) df.sort_index(inplace=True) df["time"] = pd.date_range('1/1/2011 00-01-00', periods=df.shape[0], freq='T') return df def createdir(self, path): '''创建输出目录 ''' if not os.path.exists(path): os.makedirs(path) return def setmarket(self): '''设置商品市场代码 ''' market = [] for i in range(100): market += self.api.get_instrument_info(i * 500, 500) self.market = self.api.to_df(market) return self.market def connect(self): if self.isstock: self.api = TdxHq_API(heartbeat=self.heartbeat) port = 7709 TDX_IP_SETS = self.TDX_IP_SETS_STOCK else: self.api = TdxExHq_API(heartbeat=self.heartbeat) port = 7727 TDX_IP_SETS = self.TDX_IP_SETS for ip in TDX_IP_SETS: try: if self.api.connect(ip, port): return except: pass def disconnect(self): self.api.disconnect() def getdata(self, product="ICL8", market=47, number=80000, pn=400): if product[0] in ["0", "3", "6"]: info = self.fetch_get_stock_xdxr(product) data = self.getdata_stock(product, number=number, pn=pn) data.drop(data[data["close"] <= 0].index) df = self.qfq(data, info) elif product[0] in [ "8", ]: df = self.getdata_block_index(product, market, number=number) else: df = self.getdata_future(product, market, number=number) return df def getdata_block_index(self, code="000001", market=1, number=30000, pn=500): data = [] start = False for i in range(int(number / pn) + 1): temp = self.api.get_index_bars(self.datatype, market, code, (int(number / pn) - i) * pn, pn) if temp and len(temp) > 0: start = True if start and (not temp or len(temp) < pn): for _ in range(3): temp = self.api.get_index_bars(self.datatype, market, code, (int(number / pn) - i) * pn, pn) if len(temp) < pn: print(111111111111, pn - len(temp)) else: break try: data += temp except: self.connect() df = self.api.to_df(data)[["open", "close", "high", "low", "datetime"]] df.set_index("datetime", inplace=True, drop=False) return df def getdata_future(self, product="ICL8", market=47, number=80000, pn=400): data = [] start = False for i in range(int(number / pn) + 1): temp = self.api.get_instrument_bars(self.datatype, market, product, (int(number / pn) - i) * pn, pn) if temp and len(temp) > 0: start = True if start and (not temp or len(temp) < pn): for _ in range(3): temp = self.api.get_instrument_bars( self.datatype, market, product, (int(number / pn) - i) * pn, pn) try: if len(temp) < pn: print(111111111111, pn - len(temp)) else: break except: self.connect() try: data += temp except: self.connect() df = self.api.to_df(data)[["open", "close", "high", "low", "datetime"]] df.set_index("datetime", inplace=True, drop=False) return df def set_main_rate(self, df_m, product="RBL8", f="MainContract.csv"): '''商品主月除权 ''' df = pd.read_csv(f, encoding="gb2312") df_p = df[df["ContractCode"] == product[:-2]] lstr = " 15:00" df_p = df_p.assign( datetime=df_p['EndDate'].apply(lambda x: str(x)[0:10] + lstr)) df_p.set_index("datetime", inplace=True) df_p.fillna(0, inplace=True) df_m.loc[:, "date"] = df_m["datetime"].apply(lambda x: str(x)[0:10]) filt = df_m.index.isin(df_p.index) df_m.loc[filt, "OpenPrice"] = df_p["OpenPrice"] df_m.loc[filt, "Term"] = df_p["Term"] df_m["OpenPrice"].fillna(method="bfill", inplace=True) df_m["Term"].fillna(method="bfill", inplace=True) df_m.dropna(inplace=True) filt = (df_m["OpenPrice"]!=df_m["open"])&\ (df_m["date"]>df_m["date"].shift(1))&\ (abs(1-df_m["open"]/df_m["OpenPrice"])>0.008)&\ (df_m["OpenPrice"]>0) df_m.loc[filt, "change"] = 1 df_m.loc[:, "adj"] = 1 rst = df_m[df_m["change"] > 0] rst = (rst["Term"] != rst["Term"].shift(1)) filt = df_m.index.isin(rst[rst > 0].index) df_m.loc[filt, "adj"] = df_m["open"] / df_m["OpenPrice"] df_m.loc[:, 'adj'] = df_m["adj"].shift(-1) df_m.loc[:, 'adj'] = df_m["adj"][::-1].cumprod() print(df_m[(df_m["change"] > 0) | (df_m["change"].shift(-1) > 0)][[ "close", "adj", "open", "OpenPrice", "Term" ]]) df_m.loc[:, 'open'] = df_m['open'] * df_m['adj'] df_m.loc[:, 'high'] = df_m['high'] * df_m['adj'] df_m.loc[:, 'low'] = df_m['low'] * df_m['adj'] df_m.loc[:, 'close'] = df_m['close'] * df_m['adj'] return df_m def getdata_stock(self, code="000001", number=30000, pn=500): market = self._select_market_code(code) data = [] for i in range(int(number / pn) + 1): data += self.api.get_security_bars(self.datatype, market, code, (int(number / pn) - i) * pn, pn) df = self.api.to_df(data)[["open", "close", "high", "low", "datetime"]] df.set_index("datetime", inplace=True, drop=False) return df def _select_market_code(self, code): code = str(code) if code[0] in ['5', '6', '9'] or code[:3] in [ "009", "126", "110", "201", "202", "203", "204" ]: return 1 return 0 def fetch_get_stock_xdxr(self, code): '除权除息' market_code = self._select_market_code(code) category = { '1': '除权除息', '2': '送配股上市', '3': '非流通股上市', '4': '未知股本变动', '5': '股本变化', '6': '增发新股', '7': '股份回购', '8': '增发新股上市', '9': '转配股上市', '10': '可转债上市', '11': '扩缩股', '12': '非流通股缩股', '13': '送认购权证', '14': '送认沽权证' } data = self.api.to_df(self.api.get_xdxr_info(market_code, code)) if len(data) >= 1: data = data\ .assign(date=pd.to_datetime(data[['year', 'month', 'day']]))\ .drop(['year', 'month', 'day'], axis=1)\ .assign(category_meaning=data['category'].apply(lambda x: category[str(x)]))\ .assign(code=str(code))\ .rename(index=str, columns={'panhouliutong': 'liquidity_after', 'panqianliutong': 'liquidity_before', 'houzongguben': 'shares_after', 'qianzongguben': 'shares_before'})\ .set_index('date', drop=False, inplace=False) if self.datatype == 0: lstr = " 09:35" elif self.datatype == 1: lstr = " 09:45" elif self.datatype == 2: lstr = " 10:00" elif self.datatype == 3: lstr = " 10:30" elif self.datatype == 4: lstr = " 15:00" elif self.datatype == 7: lstr = " 09:31" return data.assign( date=data['date'].apply(lambda x: str(x)[0:10] + lstr)) else: return None def qfq(self, data, xdxr_data): '''data: 除权前数据 info:除权信息 ''' start = data.index[0] if xdxr_data is not None: info = xdxr_data[xdxr_data["category"] == 1] info.set_index("date", inplace=True) df = pd.concat( [data, info[['fenhong', 'peigu', 'peigujia', 'songzhuangu']]], axis=1).fillna(0) df['preclose'] = (df['close'].shift(1) * 10 - df['fenhong'] + df['peigu'] * df['peigujia']) / ( 10 + df['peigu'] + df['songzhuangu']) df['adj'] = (df['preclose'].shift(-1) / df['close']).fillna(1)[::-1].cumprod() df['open'] = df['open'] * df['adj'] df['high'] = df['high'] * df['adj'] df['low'] = df['low'] * df['adj'] df['close'] = df['close'] * df['adj'] df['preclose'] = df['preclose'] * df['adj'] else: df['preclose'] = df['close'].shift(1) df['adj'] = 1 return df[start:] def macdhandle(self, df, p5=5, p15=15, p30=30, p60=60, p240=240, pweek=1200, macd_f=12, macd_s=26, macd_m=9): df.loc[:, "number"] = range(df.shape[0]) for i in [p5, p15, p30, p60, p240, pweek]: pres = str(i) + "_" #计算各周期初始macd df.loc[::i, pres + "fEMA_mark"] = df["close"][::i].ewm( adjust=False, span=macd_f).mean() df.loc[::i, pres + "sEMA_mark"] = df["close"][::i].ewm( adjust=False, span=macd_s).mean() df.loc[::i, pres + "DIFF_mark"] = df[pres + "fEMA_mark"] - df[pres + "sEMA_mark"] df[pres + "fEMA_mark"].fillna(method="ffill", inplace=True) df[pres + "sEMA_mark"].fillna(method="ffill", inplace=True) df[pres + "DIFF_mark"].fillna(method="ffill", inplace=True) df[pres + "fEMA"] = (df[pres + "fEMA_mark"] * (macd_f - 1) + df["close"] * 2) / (macd_f + 1) df[pres + "sEMA"] = (df[pres + "sEMA_mark"] * (macd_s - 1) + df["close"] * 2) / (macd_s + 1) df.loc[::i, pres + "fEMA"] = df[pres + "fEMA_mark"] df.loc[::i, pres + "sEMA"] = df[pres + "sEMA_mark"] df.loc[:, pres + "DIFF"] = df[pres + "fEMA"] - df[pres + "sEMA"] df.loc[::i, pres + "DEA_mark"] = df[pres + "DIFF"][::i].ewm( adjust=False, span=macd_m).mean() df[pres + "DEA_mark"].fillna(method="ffill", inplace=True) df[pres + "DEA"] = (df[pres + "DEA_mark"] * (macd_m - 1) + df[pres + "DIFF"] * 2) / (macd_m + 1) df.loc[::i, pres + "DEA"] = df[pres + "DEA_mark"] df.loc[:, pres + "MACD"] = 2 * (df[pres + "DIFF"] - df[pres + "DEA"]) df.loc[:, "DIF"], df.loc[:, "DEA"], df.loc[:, "MACD"] = talib.MACD( df.close.values) df.dropna(inplace=True) #丢弃前面NA数据 return df def getblockstock(self, block="沪深300"): '''股票版块对应股票列表 ''' df = self.api.to_df(self.api.get_and_parse_block_info("block.dat")) stocks = list(df[df["blockname"] == block]["code"]) return stocks