def get_stock_count(): api = TdxHq_API() if api.connect('119.147.212.81', 7709): print(api.get_security_count(0)) #参数为市场代码 print(api.get_security_list(0, 0)) #第一个参数为市场代码,第二个参数为起始位置 print(api.get_security_count(1)) print(api.to_df(api.get_security_list(1, 0))) api.disconnect()
def QA_fetch_get_stock_list(type_='stock', ip=best_ip['stock'], port=7709): api = TdxHq_API() with api.connect(ip, port): data = pd.concat([pd.concat([api.to_df(api.get_security_list(j, i * 1000)).assign(sse='sz' if j == 0 else 'sh').set_index( ['code', 'sse'], drop=False) for i in range(int(api.get_security_count(j) / 1000) + 1)], axis=0) for j in range(2)], axis=0) #data.code = data.code.apply(int) sz = data.query('sse=="sz"') sh = data.query('sse=="sh"') sz = sz.assign(sec=sz.code.apply(for_sz)) sh = sh.assign(sec=sh.code.apply(for_sh)) if type_ in ['stock', 'gp']: return pd.concat([sz, sh]).query('sec=="stock_cn"').sort_index().assign(name=data['name'].apply(lambda x: str(x)[0:6])) elif type_ in ['index', 'zs']: return pd.concat([sz, sh]).query('sec=="index_cn"').sort_index().assign(name=data['name'].apply(lambda x: str(x)[0:6])) #.assign(szm=data['name'].apply(lambda x: ''.join([y[0] for y in lazy_pinyin(x)])))\ #.assign(quanpin=data['name'].apply(lambda x: ''.join(lazy_pinyin(x)))) elif type_ in ['etf', 'ETF']: return pd.concat([sz, sh]).query('sec=="etf_cn"').sort_index().assign(name=data['name'].apply(lambda x: str(x)[0:6])) else: return data.assign(code=data['code'].apply(lambda x: str(x))).assign(name=data['name'].apply(lambda x: str(x)[0:6]))
def QA_fetch_get_stock_list(type_='stock', ip=best_ip, port=7709): api = TdxHq_API() with api.connect(ip, port): data = pd.concat([pd.concat([api.to_df(api.get_security_list(j, i * 1000)).assign(sse='sz' if j == 0 else 'sh').set_index( ['code', 'sse'], drop=False) for i in range(int(api.get_security_count(j) / 1000) + 1)], axis=0) for j in range(2)], axis=0) if type_ in ['stock', 'gp']: return pd.concat([data[data['sse'] == 'sz'][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 10000 <= 30][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 100000 != 2], data[data['sse'] == 'sh'][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 100000 == 6]]).assign(code=data['code'].apply(lambda x: str(x))) #.assign(szm=data['name'].apply(lambda x: ''.join([y[0] for y in lazy_pinyin(x)])))\ #.assign(quanpin=data['name'].apply(lambda x: ''.join(lazy_pinyin(x)))) elif type_ in ['index', 'zs']: return pd.concat([data[data['sse'] == 'sz'][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 1000 >= 399], data[data['sse'] == 'sh'][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 1000 == 0]]) \ .sort_index()\ .assign(code=data['code'].apply(lambda x: str(x))) #.assign(szm=data['name'].apply(lambda x: ''.join([y[0] for y in lazy_pinyin(x)])))\ #.assign(quanpin=data['name'].apply(lambda x: ''.join(lazy_pinyin(x)))) elif type_ in ['etf', 'ETF']: return pd.concat([data[data['sse'] == 'sz'][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 10000 == 15], data[data['sse'] == 'sh'][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 10000 == 51]]).sort_index().assign(code=data['code'].apply(lambda x: str(x)))\ #.assign(szm=data['name'].apply(lambda x: ''.join([y[0] for y in lazy_pinyin(x)])))\ #.assign(quanpin=data['name'].apply(lambda x: ''.join(lazy_pinyin(x)))) else: return data.assign(code=data['code'].apply(lambda x: str(x)))
def __hq_list(self, market): assert self.__hq.qsize() > 0 api = TdxHq_API() ip, port = self.__hq.get() with api.connect(ip, port): df = list() for start in range(0, api.get_security_count(market=market), 1000): df.append(api.to_df(api.get_security_list(market, start))) api.disconnect() self.__hq.put((ip, port)) df = pandas.concat(df, sort=False).assign(sse=market) df = df[[ 'code', 'volunit', 'decimal_point', 'name', 'pre_close', 'sse' ]].dropna() df = df.assign(sse='sh' if market == CODE_MARKET_SH else 'sz', sec=get_code_type(df.code.tolist(), market)) return df return None
def QA_fetch_get_stock_list(type_='stock', ip=best_ip, port=7709): api = TdxHq_API() with api.connect(ip, port): data = pd.concat([pd.concat([api.to_df(api.get_security_list(j, i * 1000)).assign(sse='sz' if j == 0 else 'sh').set_index( ['code', 'sse'], drop=False) for i in range(int(api.get_security_count(j) / 1000) + 1)], axis=0) for j in range(2)], axis=0) if type_ in ['stock', 'gp']: return pd.concat([data[data['sse'] == 'sz'][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 10000 <= 30][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 100000 != 2], data[data['sse'] == 'sh'][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 100000 == 6]]).assign(code=data['code'].apply(lambda x: str(x))) elif type_ in ['index', 'zs']: return pd.concat([data[data['sse'] == 'sz'][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 1000 >= 399], data[data['sse'] == 'sh'][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 1000 == 0]]).sort_index().assign(code=data['code'].apply(lambda x: str(x))) elif type_ in ['etf', 'ETF']: return pd.concat([data[data['sse'] == 'sz'][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 10000 == 15], data[data['sse'] == 'sh'][data.assign(code=data['code'].apply(lambda x: int(x)))['code'] // 10000 == 51]]).sort_index().assign(code=data['code'].apply(lambda x: str(x))) else: return data.assign(code=data['code'].apply(lambda x: str(x)))
class tdxApi: def __init__(self): self.ipset = [(v[1], v[2]) for v in hq_hosts] random.shuffle(self.ipset) self.ippool = AvailableIPPool(TdxHq_API, self.ipset) self.primary_ip, _ = self.ippool.sync_get_top_n(2) self.api = TdxHq_API(multithread=True, heartbeat=True, auto_retry=True) def connect(self): self.api.connect(self.primary_ip[0], self.primary_ip[1]) def release(self): self.api.disconnect() def getQuotes(self, stock_list): datas = self.api.get_security_quotes(stock_list) if datas is None: return None else: return self.api.to_df(datas) def getList(self, market, index): df = self.api.to_df(self.api.get_security_list(market, index)) return df def getCount(self, market=0): return self.api.get_security_count(market) def getMinQuotes(self, market, stock_code): datas = self.api.get_minute_time_data(market, stock_code) if datas is None: return None return self.api.to_df(datas) def getMinQuotesHis(self, market, stock_code, date): datas = self.api.get_history_minute_time_data(market, stock_code, date) if datas is None: return None return self.api.to_df(datas)
def __do_fetch_security_list(self, api: TdxHq_API, exchange: Exchange): arr = [] start = 0 ex_nbr = map_exchange_to_tdx_number[exchange] count = api.get_security_count(ex_nbr) while start < count: tmp = api.get_security_list(ex_nbr, start) if tmp is None: break sz = len(tmp) if sz < 1: break start += sz arr.extend(tmp) print('total fetched rows: {}'.format(start)) df = api.to_df(arr).assign(ex=exchange.name) df = df.assign( stype=df.code.apply(exchange.code_to_type_str) ).set_index(['code', 'ex'], drop=True) return df[df.stype.notnull()]
def get_fast_hq_ip(self) -> (str, int): fast_hq_ip_dict = {} hq_api = TdxHq_API() for name, ip in self.hq_ips_dict.items(): # 东莞电信主站的port和别的不一样 if name == "东莞电信主站": hq_port = 7721 else: hq_port = self.hq_port with hq_api.connect(ip, hq_port): start_time = time.time() instrument_count = hq_api.get_security_count(0) cost_time = time.time() - start_time self.log.write_log(f"{name}({ip}), time: {cost_time:.3f}s, response: {instrument_count}") fast_hq_ip_dict[f"{ip}:{hq_port}"] = cost_time ip_str, port_str = min(fast_hq_ip_dict, key=fast_hq_ip_dict.get).split(":") self.log.write_log(f"-" * 50) self.log.write_log(f"Select ({ip_str} : {port_str})") self.log.write_log(f"-" * 50) return ip_str, int(port_str)
def QA_fetch_get_index_list(ip=None, port=None): """获取指数列表 Keyword Arguments: ip {[type]} -- [description] (default: {None}) port {[type]} -- [description] (default: {None}) Returns: [type] -- [description] """ ip, port = get_mainmarket_ip(ip, port) api = TdxHq_API() with api.connect(ip, port): data = pd.concat([pd.concat([api.to_df(api.get_security_list(j, i * 1000)).assign(sse='sz' if j == 0 else 'sh').set_index( ['code', 'sse'], drop=False) for i in range(int(api.get_security_count(j) / 1000) + 1)], axis=0) for j in range(2)], axis=0) #data.code = data.code.apply(int) sz = data.query('sse=="sz"') sh = data.query('sse=="sh"') sz = sz.assign(sec=sz.code.apply(for_sz)) sh = sh.assign(sec=sh.code.apply(for_sh)) return pd.concat([sz, sh]).query('sec=="index_cn"').sort_index().assign(name=data['name'].apply(lambda x: str(x)[0:6]))
def QA_fetch_get_stock_list(type_='stock', ip=None, port=None): global best_ip if ip is None and port is None and best_ip['stock']['ip'] is None and best_ip['stock']['port'] is None: best_ip = select_best_ip() ip = best_ip['stock']['ip'] port = best_ip['stock']['port'] elif ip is None and port is None and best_ip['stock']['ip'] is not None and best_ip['stock']['port'] is not None: ip = best_ip['stock']['ip'] port = best_ip['stock']['port'] else: pass api = TdxHq_API() with api.connect(ip, port): data = pd.concat([pd.concat([api.to_df(api.get_security_list(j, i * 1000)).assign(sse='sz' if j == 0 else 'sh').set_index( ['code', 'sse'], drop=False) for i in range(int(api.get_security_count(j) / 1000) + 1)], axis=0) for j in range(2)], axis=0) #data.code = data.code.apply(int) sz = data.query('sse=="sz"') sh = data.query('sse=="sh"') sz = sz.assign(sec=sz.code.apply(for_sz)) sh = sh.assign(sec=sh.code.apply(for_sh)) if type_ in ['stock', 'gp']: return pd.concat([sz, sh]).query('sec=="stock_cn"').sort_index().assign(name=data['name'].apply(lambda x: str(x)[0:6])) elif type_ in ['index', 'zs']: return pd.concat([sz, sh]).query('sec=="index_cn"').sort_index().assign(name=data['name'].apply(lambda x: str(x)[0:6])) #.assign(szm=data['name'].apply(lambda x: ''.join([y[0] for y in lazy_pinyin(x)])))\ #.assign(quanpin=data['name'].apply(lambda x: ''.join(lazy_pinyin(x)))) elif type_ in ['etf', 'ETF']: return pd.concat([sz, sh]).query('sec=="etf_cn"').sort_index().assign(name=data['name'].apply(lambda x: str(x)[0:6])) else: return data.assign(code=data['code'].apply(lambda x: str(x))).assign(name=data['name'].apply(lambda x: str(x)[0:6]))
class Engine: def __init__(self, *args, **kwargs): if kwargs.pop('best_ip', False): self.ip = self.best_ip else: self.ip = '14.17.75.71' self.ip = kwargs.pop('ip', '14.17.75.71') self.thread_num = kwargs.pop('thread_num', 1) if not PY2 and self.thread_num != 1: self.use_concurrent = True else: self.use_concurrent = False self.api = TdxHq_API(args, kwargs) if self.use_concurrent: self.apis = [ TdxHq_API(args, kwargs) for i in range(self.thread_num) ] self.executor = ThreadPoolExecutor(self.thread_num) def connect(self): self.api.connect(self.ip) if self.use_concurrent: for api in self.apis: api.connect(self.ip) return self def __enter__(self): return self def exit(self): self.api.disconnect() if self.use_concurrent: for api in self.apis: api.disconnect() def __exit__(self, exc_type, exc_val, exc_tb): self.api.disconnect() if self.use_concurrent: for api in self.apis: api.disconnect() def quotes(self, code): code = [code] if not isinstance(code, list) else code code = self.security_list[self.security_list.code.isin( code)].index.tolist() data = [ self.api.to_df( self.api.get_security_quotes(code[80 * pos:80 * (pos + 1)])) for pos in range(int(len(code) / 80) + 1) ] return pd.concat(data) # data = data[['code', 'open', 'high', 'low', 'price']] # data['datetime'] = datetime.datetime.now() # return data.set_index('code', drop=False, inplace=False) def stock_quotes(self): code = self.stock_list.index.tolist() if self.use_concurrent: res = { self.executor.submit(self.apis[pos % self.thread_num].get_security_quotes, code[80 * pos:80 * (pos + 1)]) \ for pos in range(int(len(code) / 80) + 1)} return pd.concat([self.api.to_df(dic.result()) for dic in res]) else: data = [ self.api.to_df( self.api.get_security_quotes(code[80 * pos:80 * (pos + 1)])) for pos in range(int(len(code) / 80) + 1) ] return pd.concat(data) @lazyval def security_list(self): return pd.concat([ pd.concat([ self.api.to_df(self.api.get_security_list( j, i * 1000)).assign(sse=0 if j == 0 else 1).set_index( ['sse', 'code'], drop=False) for i in range(int(self.api.get_security_count(j) / 1000) + 1) ], axis=0) for j in range(2) ], axis=0) @lazyval def stock_list(self): aa = map(stock_filter, self.security_list.index.tolist()) return self.security_list[list(aa)] @lazyval def best_ip(self): return select_best_ip() @lazyval def concept(self): return self.api.to_df( self.api.get_and_parse_block_info(TDXParams.BLOCK_GN)) @lazyval def index(self): return self.api.to_df( self.api.get_and_parse_block_info(TDXParams.BLOCK_SZ)) @lazyval def fengge(self): return self.api.to_df( self.api.get_and_parse_block_info(TDXParams.BLOCK_FG)) @lazyval def customer_block(self): return CustomerBlockReader().get_df(CUSTOMER_BLOCK_PATH) @lazyval def gbbq(self): df = GbbqReader().get_df(GBBQ_PATH).query('category == 1') df['datetime'] = pd.to_datetime(df['datetime'], format='%Y%m%d') return df def get_security_type(self, code): if code in self.security_list.code.values: return self.security_list[self.security_list.code == code]['sse'].as_matrix()[0] else: raise SecurityNotExists() def get_security_bars(self, code, freq, index=False): if index: exchange = self.get_security_type(code) func = self.api.get_index_bars else: exchange = get_stock_type(code) func = self.api.get_security_bars df = pd.DataFrame() if freq in ['1d', 'day']: freq = 9 elif freq in ['1m', 'min']: freq = 8 else: raise Exception("1d and 1m frequency supported only") res = [] start = 0 while True: data = func(freq, exchange, code, start, 800) if not data: break res = data + res start += 800 df = self.api.to_df(res).drop( ['year', 'month', 'day', 'hour', 'minute'], axis=1) df['datetime'] = pd.to_datetime(df.datetime) df['code'] = code return df.set_index('datetime') def _get_transaction(self, code, date): res = [] start = 0 while True: data = self.api.get_history_transaction_data( get_stock_type(code), code, start, 2000, date) if not data: break start += 2000 res = data + res if len(res) == 0: return pd.DataFrame() df = self.api.to_df(res).assign(date=date) df.index = pd.to_datetime(str(date) + " " + df["time"]) df['code'] = code return df.drop("time", axis=1) def time_and_price(self, code): start = 0 res = [] exchange = self.get_security_type(code) while True: data = self.api.get_transaction_data(exchange, code, start, 2000) if not data: break res = data + res start += 2000 df = self.api.to_df(res) df.time = pd.to_datetime( str(pd.to_datetime('today').date()) + " " + df['time']) df.loc[0, 'time'] = df.time[1] return df.set_index('time') @classmethod def minute_bars_from_transaction(cls, transaction, freq): if transaction.empty: return pd.DataFrame() data = transaction['price'].resample(freq, label='right', closed='left').ohlc() data['volume'] = transaction['vol'].resample(freq, label='right', closed='left').sum() data['code'] = transaction['code'][0] return fillna(data) def get_k_data(self, code, start, end, freq): if isinstance(start, str) or isinstance(end, str): start = pd.Timestamp(start) end = pd.Timestamp(end) sessions = pd.date_range(start, end) trade_days = map(int, sessions.strftime("%Y%m%d")) if freq == '1m': freq = '1 min' if freq == '1d': freq = '24 H' res = [] for trade_day in trade_days: df = Engine.minute_bars_from_transaction( self._get_transaction(code, trade_day), freq) if df.empty: continue res.append(df) if len(res) != 0: return pd.concat(res) return pd.DataFrame()
class StdQuotes(object): """股票市场实时行情""" bestip = ('47.103.48.45', 7709) def __init__(self, **kwargs): try: default = settings.get('SERVER').get('HQ')[0] self.bestip = config.get('BESTIP').get('HQ', default) except ValueError: self.config = None self.client = TdxHq_API(**kwargs) def traffic(self): with self.client.connect(*self.bestip): return self.client.get_traffic_stats() def quotes(self, symbol=[]): ''' 获取实时日行情数据 :param symbol: 股票代码 :return: pd.dataFrame or None ''' logger.debug(type(logger)) if type(symbol) is str: symbol = [symbol] with self.client.connect(*self.bestip): symbol = get_stock_markets(symbol) result = self.client.get_security_quotes(symbol) return to_data(result) def bars(self, symbol='000001', frequency='9', start='0', offset='100'): ''' 获取实时日K线数据 :param symbol: 股票代码 :param frequency: 数据类别 :param market: 证券市场 :param start: 开始位置 :param offset: 每次获取条数 :return: pd.dataFrame or None ''' with self.client.connect(*self.bestip): market = get_stock_market(symbol) result = self.client.get_security_bars(int(frequency), int(market), str(symbol), int(start), int(offset)) return to_data(result) def stock_count(self, market=MARKET_SH): ''' 获取市场股票数量 :param market: 股票市场代码 sh 上海, sz 深圳 :return: pd.dataFrame or None ''' with self.client.connect(*self.bestip): result = self.client.get_security_count(market=market) return result def stocks(self, market=MARKET_SH): ''' 获取股票列表 :param market: :return: ''' with self.client.connect(*self.bestip): counts = self.client.get_security_count(market=market) stocks = None for start in tqdm(range(0, counts, 1000)): result = self.client.get_security_list(market=market, start=start) stocks = pandas.concat( [stocks, to_data(result)], ignore_index=True) if start > 1 else to_data(result) return stocks def index_bars(self, symbol='000001', frequency='9', start='0', offset='100'): ''' 获取指数k线 :param symbol: :param frequency: :param start: :param offset: :return: ''' with self.client.connect(*self.bestip): market = get_stock_market(symbol) result = self.client.get_index_bars(frequency=frequency, market=market, code=symbol, start=start, count=offset) return to_data(result) def minute(self, symbol=''): ''' 获取实时分时数据 :param market: 证券市场 :param symbol: 股票代码 :return: pd.DataFrame ''' with self.client.connect(*self.bestip): market = get_stock_market(symbol) result = self.client.get_minute_time_data(market=market, code=symbol) return to_data(result) def minutes(self, symbol='', date='20191023'): ''' 分时历史数据 :param market: :param symbol: :param date: :return: pd.dataFrame or None ''' with self.client.connect(*self.bestip): market = get_stock_market(symbol) result = self.client.get_history_minute_time_data(market=market, code=symbol, date=date) return to_data(result) def transaction(self, symbol='', start=0, offset=10): ''' 查询分笔成交 :param market: 市场代码 :param symbol: 股票代码 :param start: 起始位置 :param offset: 请求数量 :return: pd.dataFrame or None ''' with self.client.connect(*self.bestip): market = get_stock_market(symbol) result = self.client.get_transaction_data(int(market), symbol, int(start), int(offset)) return to_data(result) def transactions(self, symbol='', start=0, offset=10, date='20170209'): ''' 查询历史分笔成交 参数:市场代码, 股票代码,起始位置,日期 数量 如: 0,000001,0,10,20170209 :param symbol: 股票代码 :param start: 起始位置 :param offset: 数量 :param date: 日期 :return: pd.dataFrame or None ''' with self.client.connect(*self.bestip): market = get_stock_market(symbol, string=False) result = self.client.get_history_transaction_data(market=market, code=symbol, start=start, count=offset, date=date) return to_data(result) def F10C(self, symbol=''): ''' 查询公司信息目录 :param market: 市场代码 :param symbol: 股票代码 :return: pd.dataFrame or None ''' with self.client.connect(*self.bestip): market = get_stock_market(symbol) result = self.client.get_company_info_category(int(market), symbol) return result def F10(self, symbol='', name=''): ''' 读取公司信息详情 :param name: 公司 F10 标题 :param symbol: 股票代码 :return: pd.dataFrame or None ''' with self.client.connect(*self.bestip): result = {} market = get_stock_market(symbol, string=False) frequency = self.client.get_company_info_category( int(market), symbol) if name: for x in frequency: if x['name'] == name: return self.client.get_company_info_content( market=market, code=symbol, filename=x['filename'], start=x['start'], length=x['length']) for x in frequency: result[x['name']] = self.client.get_company_info_content( market=market, code=symbol, filename=x['filename'], start=x['start'], length=x['length']) else: pass return result def xdxr(self, symbol=''): ''' 读取除权除息信息 :param market: 市场代码 :param symbol: 股票代码 :return: pd.dataFrame or None ''' with self.client.connect(*self.bestip): market = get_stock_market(symbol) result = self.client.get_xdxr_info(int(market), symbol) return to_data(result) def finance(self, symbol='000001'): ''' 读取财务信息 :param symbol: :return: ''' with self.client.connect(*self.bestip): market = get_stock_market(symbol) result = self.client.get_finance_info(market=market, code=symbol) return to_data(result) def k(self, symbol='', begin=None, end=None): ''' 读取k线信息 :param symbol: :param begin: 开始日期 :param end: 截止日期 :return: pd.dataFrame or None ''' with self.client.connect(*self.bestip): result = self.client.get_k_data(symbol, begin, end) return to_data(result) def index(self, symbol='000001', market=MARKET_SH, frequency='9', start=1, offset=2): ''' 获取指数k线 K线种类: - 0 5分钟K线 - 1 15分钟K线 - 2 30分钟K线 - 3 1小时K线 - 4 日K线 - 5 周K线 - 6 月K线 - 7 1分钟 - 8 1分钟K线 - 9 日K线 - 10 季K线 - 11 年K线 :param symbol: 股票代码 :param frequency: 数据类别 :param market: 证券市场 :param start: 开始位置 :param offset: 每次获取条数 :return: pd.dataFrame or None ''' with self.client.connect(*self.bestip): result = self.client.get_index_bars(int(frequency), int(market), str(symbol), int(start), int(offset)) return to_data(result) def block(self, tofile="block.dat"): ''' 获取证券板块信息 :param tofile: :return: pd.dataFrame or None ''' with self.client.connect(*self.bestip): result = self.client.get_and_parse_block_info(tofile) return to_data(result)
class PytdxApi: """TDX数据服务类""" def __init__(self): """Constructor""" self.connect_status: bool = False self.login_status: bool = False self.hq_api = None # 行情API self.conc_code_num = 50 # 并发获取行情的股票个数 # 行情订阅 self.active = False self.run_subscribe = Thread(target=self.get_realtime_data) self.symbols = list() self.symbols_split = list() def connect_api(self, host: str = "", port: int = 0): """连接行情api""" # 连接行情API并检查连接情况 try: if not self.connect_status: self.hq_api = TdxHq_API() self.hq_api.connect(host, port) self.connect_status = True self.login_status = True self.subscribe_start() except Exception as e: return e def get_realtime_quotes(self, quotes_list: list): """获取实时行情数据""" data = self.hq_api.get_security_quotes(quotes_list) return data def get_realtime_data(self): """获取实时行情切片""" try: while self.active: if not self.symbols_split: sleep(1) continue data = list() for symbols in self.symbols_split: d = self.get_realtime_quotes(symbols) data.extend(d) self.on_tick_data(data) sleep(2) except: error = dict() error["error_id"] = "pytdx" error["error_msg"] = "行情订阅失败" self.on_error(error) def get_transaction_count(self, market: int) -> int: """ 查询市场标的数量 """ return self.hq_api.get_security_count(market) def get_transaction_list(self, market: int, start: int) -> list: """查询市场标的列表""" return self.hq_api.get_security_list(market, start) def subscribe_start(self): """启动行情订阅""" self.active = True self.run_subscribe.start() def subscribe(self, symbol: Any): """订阅行情数据""" if isinstance(symbol, tuple): if symbol not in self.symbols: self.symbols.append(symbol) elif isinstance(symbol, list): for s in symbol: if s not in self.symbols: self.symbols.append(s) else: error = dict() error["error_id"] = "pytdx" error["error_msg"] = f"订阅标的代码格式不正确{symbol}" self.on_error(error) return symbol_split = self.get_code_split() self.symbols_split = copy(symbol_split) def subscribe_close(self): """关闭订阅""" if self.active: self.active = False self.run_subscribe.join() def get_transaction_info(self): """获取所有合约信息""" for exchange in list(exchange_map.values()): count = self.get_transaction_count(exchange) for c in range(0, count, 1000): symbols = self.get_transaction_list(exchange, c) for symbol in symbols: symbol["exchange"] = exchange if symbol["code"][:2] in ["60", "30", "688", "00"]: symbol["product"] = 3 else: symbol["product"] = 2 self.on_contract_info(symbol, False) self.on_contract_info({"exchange": exchange}, True) def get_all_stock(self): """获取所有股票数据""" stocks = list() for exchange in list(exchange_map.values()): count = self.get_transaction_count(exchange) for s in range(0, count, 1000): d = self.get_transaction_list(exchange, s) stocks.extend(d) l = len(stocks) - 1 for i, stock in enumerate(stocks): if stock["code"][:1] in ["60", "30", "688", "00"]: if i == l: self.on_contract_info(stock, True) else: self.on_contract_info(stock, False) def on_contract_info(self, data: dict, last: bool) -> None: """""" pass def on_tick_data(self, data): """订阅数据处理""" pass def on_error(self, err): """接口错误处理""" pass @staticmethod def generate_symbols(symbols: list): """组装symbols数据,pytdx接收的是以市场代码和标的代码组成的元祖的list""" new_symbols = [] for symbol in symbols: code, exchange = symbol.split(".") new_symbol = (exchange_map[exchange], code) new_symbols.append(new_symbol) return new_symbols @staticmethod def get_fast_ip(): """获取最快IP""" host = "210.51.39.201" port = 7709 return host, port @staticmethod def check_symbol(symbol: str): """检查标的格式""" if symbol: code, market = symbol.split(".") market = exchange_map.get(market) return code, market else: return False def get_code_split(self): """获得切割好的股票代码段""" code_split_list = [] for i in range(0, len(self.symbols) + 1, self.conc_code_num): code_split = self.symbols[i : i + self.conc_code_num] code_split_list.append(code_split) return code_split_list def exit(self): """数据服务关闭""" # 关闭订阅 self.subscribe_close() # 关闭接口 self.login_status = False self.connect_status = False self.hq_api.disconnect() self.hq_api = None
def test_all_functions(multithread, heartbeat, auto_retry, raise_exception): api = TdxHq_API(multithread=multithread, heartbeat=heartbeat, auto_retry=auto_retry, raise_exception=raise_exception) with api.connect(time_out=30): log.info("获取股票行情") stocks = api.get_security_quotes([(0, "000001"), (1, "600300")]) assert stocks is not None assert type(stocks) is list # 方法2 stocks = api.get_security_quotes(0, "000001") assert stocks is not None assert type(stocks) is list # 方法3 stocks = api.get_security_quotes((0, "000001")) assert stocks is not None assert type(stocks) is list log.info("获取k线") data = api.get_security_bars(9, 0, '000001', 4, 3) assert data is not None assert type(data) is list assert len(data) == 3 log.info("获取 深市 股票数量") assert api.get_security_count(0) > 0 log.info("获取股票列表") stocks = api.get_security_list(1, 0) assert stocks is not None assert type(stocks) is list assert len(stocks) > 0 log.info("获取指数k线") data = api.get_index_bars(9, 1, '000001', 1, 2) assert data is not None assert type(data) is list assert len(data) == 2 log.info("查询分时行情") data = api.get_minute_time_data(TDXParams.MARKET_SH, '600300') assert data is not None log.info("查询历史分时行情") data = api.get_history_minute_time_data( TDXParams.MARKET_SH, '600300', 20161209) assert data is not None assert type(data) is list assert len(data) > 0 log.info("查询分时成交") data = api.get_transaction_data(TDXParams.MARKET_SZ, '000001', 0, 30) assert data is not None assert type(data) is list log.info("查询历史分时成交") data = api.get_history_transaction_data( TDXParams.MARKET_SZ, '000001', 0, 10, 20170209) assert data is not None assert type(data) is list assert len(data) == 10 log.info("查询公司信息目录") data = api.get_company_info_category(TDXParams.MARKET_SZ, '000001') assert data is not None assert type(data) is list assert len(data) > 0 start = data[0]['start'] length = data[0]['length'] log.info("读取公司信息-最新提示") data = api.get_company_info_content( 0, '000001', '000001.txt', start, length) assert data is not None assert len(data) > 0 log.info("读取除权除息信息") data = api.get_xdxr_info(1, '600300') assert data is not None assert type(data) is list assert len(data) > 0 log.info("读取财务信息") data = api.get_finance_info(0, '000001') assert data is not None assert type(data) is OrderedDict assert len(data) > 0 log.info("日线级别k线获取函数") data = api.get_k_data('000001', '2017-07-01', '2017-07-10') assert type(data) is pd.DataFrame assert len(data) == 6 log.info("获取板块信息") data = api.get_and_parse_block_info(TDXParams.BLOCK_FG) assert data is not None assert type(data) is list assert len(data) > 0
import time api = TdxHq_API(heartbeat=True) ttsengine = pyttsx3.init() mylog.add("stock_trading_{time}.log", encoding='utf-8') from pytdx.util.best_ip import select_best_ip stock_ip = select_best_ip('stock') print(stock_ip) future_ip = select_best_ip('future') print(future_ip) # if api.connect('119.147.212.81', 7709): if api.connect(stock_ip['ip'], stock_ip['port']): szall = api.get_security_count(0) shall = api.get_security_count(1) szsecs = api.get_security_list(0, 0) shsecs = api.get_security_list(1, 0) # szStockProcess(DAY) # shStockProcess(DAY) print('全市场轮询结束.') # data = api.get_k_data(stock, '2015-01-01', '2020-01-14') # shsecdict = {} # shall = api.get_security_count(1) # for step in range(0, shall, 1000): # shsecs = api.get_security_list(1, step) # for sec in shsecs: # if sec['code'].startswith('68') or sec['code'].startswith('60'):
data = api.get_k_data('000002', '2005-07-01', '2017-07-10') data2 = api.get_xdxr_info(1, '600300') print(data2) print("获取股票行情") stocks = api.get_security_quotes([(0, "000002"), (1, "600300")]) print(stocks) print("获取k线") data = api.get_security_bars(9, 0, '000001', 4, 3) print(data) print("获取 深市 股票数量") print(api.get_security_count(0)) print("获取股票列表") stocks = api.get_security_list(1, 255) print(stocks) print("获取指数k线") data = api.get_index_bars(9, 1, '000001', 1, 2) print(data) print("查询分时行情") data = api.get_minute_time_data(1, '600300') print(data) print("查询历史分时行情") data = api.get_history_minute_time_data(1, '600300', 20161209) print(data) print("查询分时成交") data = api.get_transaction_data(1, '000002', 0, 30) print(data)
class Engine: concurrent_thread_count = 50 def __init__(self, *args, **kwargs): if 'ip' in kwargs: self.ip = kwargs.pop('ip') else: if kwargs.pop('best_ip', False): self.ip = self.best_ip else: self.ip = '14.17.75.71' if 'concurrent_thread_count' in kwargs: self.concurrent_thread_count = kwargs.pop( 'concurrent_thread_count', 50) self.thread_num = kwargs.pop('thread_num', 1) self.api = TdxHq_API(args, kwargs, raise_exception=True) def connect(self): self.api.connect(self.ip) return self def __enter__(self): return self def exit(self): self.api.disconnect() def __exit__(self, exc_type, exc_val, exc_tb): self.api.disconnect() def quotes(self, code): code = [code] if not isinstance(code, list) else code code = self.security_list[self.security_list.code.isin( code)].index.tolist() data = [ self.api.to_df( self.api.get_security_quotes(code[80 * pos:80 * (pos + 1)])) for pos in range(int(len(code) / 80) + 1) ] return pd.concat(data) # data = data[['code', 'open', 'high', 'low', 'price']] # data['datetime'] = datetime.datetime.now() # return data.set_index('code', drop=False, inplace=False) def stock_quotes(self): code = self.stock_list.index.tolist() data = [ self.api.to_df( self.api.get_security_quotes(code[80 * pos:80 * (pos + 1)])) for pos in range(int(len(code) / 80) + 1) ] return pd.concat(data) @lazyval def security_list(self): return pd.concat([ pd.concat([ self.api.to_df(self.api.get_security_list( j, i * 1000)).assign(sse=0 if j == 0 else 1).set_index( ['sse', 'code'], drop=False) for i in range(int(self.api.get_security_count(j) / 1000) + 1) ], axis=0) for j in range(2) ], axis=0) @lazyval def stock_list(self): aa = map(stock_filter, self.security_list.index.tolist()) return self.security_list[list(aa)] @lazyval def best_ip(self): return select_best_ip() @lazyval def concept(self): return self.api.to_df( self.api.get_and_parse_block_info(TDXParams.BLOCK_GN)) @lazyval def index(self): return self.api.to_df( self.api.get_and_parse_block_info(TDXParams.BLOCK_SZ)) @lazyval def fengge(self): return self.api.to_df( self.api.get_and_parse_block_info(TDXParams.BLOCK_FG)) @lazyval def block(self): return self.api.to_df( self.api.get_and_parse_block_info(TDXParams.BLOCK_DEFAULT)) @lazyval def customer_block(self): return CustomerBlockReader().get_df(CUSTOMER_BLOCK_PATH) def xdxr(self, code): df = self.api.to_df( self.api.get_xdxr_info(self.get_security_type(code), code)) if df.empty: return df df['datetime'] = pd.to_datetime((df.year * 10000 + df.month * 100 + df.day).apply(lambda x: str(x))) return df.drop(['year', 'month', 'day'], axis=1).set_index('datetime') @lazyval def gbbq(self): df = GbbqReader().get_df(GBBQ_PATH).query('category == 1') df['datetime'] = pd.to_datetime(df['datetime'], format='%Y%m%d') return df def get_security_type(self, code): if code in self.security_list.code.values: return self.security_list[self.security_list.code == code]['sse'].as_matrix()[0] else: raise SecurityNotExists() @retry(3) def get_security_bars(self, code, freq, start=None, end=None, index=False): if index: exchange = self.get_security_type(code) func = self.api.get_index_bars else: exchange = get_stock_type(code) func = self.api.get_security_bars if start: start = start.tz_localize(None) if end: end = end.tz_localize(None) if freq in ['1d', 'day']: freq = 9 elif freq in ['1m', 'min']: freq = 8 else: raise Exception("1d and 1m frequency supported only") res = [] pos = 0 while True: data = func(freq, exchange, code, pos, 800) if not data: break res = data + res pos += 800 if start and pd.to_datetime(data[0]['datetime']) < start: break try: df = self.api.to_df(res).drop( ['year', 'month', 'day', 'hour', 'minute'], axis=1) df['datetime'] = pd.to_datetime(df.datetime) df.set_index('datetime', inplace=True) if freq == 9: df.index = df.index.normalize() except ValueError: # 未上市股票,无数据 logger.warning("no k line data for {}".format(code)) # return pd.DataFrame({ # 'amount': [0], # 'close': [0], # 'open': [0], # 'high': [0], # 'low': [0], # 'vol': [0], # 'code': code # }, # index=[start] # ) return pd.DataFrame() close = [df.close.values[-1]] if start: df = df.loc[lambda df: start <= df.index] if end: df = df.loc[lambda df: df.index.normalize() <= end] if df.empty: # return pd.DataFrame({ # 'amount': [0], # 'close': close, # 'open': close, # 'high': close, # 'low': close, # 'vol': [0], # 'code': code # }, # index=[start] # ) return df else: if int(df['vol'][-1]) <= 0 and end == df.index[-1] and len( df) == 1: # 成交量为0,当天返回的是没开盘的数据 return pd.DataFrame() df['code'] = code return df def _get_transaction(self, code, date): res = [] start = 0 while True: data = self.api.get_history_transaction_data( get_stock_type(code), code, start, 2000, date) if not data: break start += 2000 res = data + res if len(res) == 0: return pd.DataFrame() df = self.api.to_df(res).assign(date=date) df.loc[0, 'time'] = df.time[1] df.index = pd.to_datetime(str(date) + " " + df["time"]) df['code'] = code return df.drop("time", axis=1) def time_and_price(self, code): start = 0 res = [] exchange = self.get_security_type(code) while True: data = self.api.get_transaction_data(exchange, code, start, 2000) if not data: break res = data + res start += 2000 df = self.api.to_df(res) df.time = pd.to_datetime( str(pd.to_datetime('today').date()) + " " + df['time']) df.loc[0, 'time'] = df.time[1] return df.set_index('time') @classmethod def minute_bars_from_transaction(cls, transaction, freq): if transaction.empty: return pd.DataFrame() mask = transaction.index < transaction.index[0].normalize( ) + pd.Timedelta('12 H') def resample(transaction): if transaction.empty: return pd.DataFrame() data = transaction['price'].resample(freq, label='right', closed='left').ohlc() data['volume'] = transaction['vol'].resample(freq, label='right', closed='left').sum() data['code'] = transaction['code'][0] return data morning = resample(transaction[mask]) afternoon = resample(transaction[~mask]) if morning.empty and afternoon.empty: return pd.DataFrame() if not afternoon.empty: morning.index.values[-1] = afternoon.index[0] - pd.Timedelta( '1 min') df = pd.concat([morning, afternoon]) return fillna(df) def _get_k_data(self, code, freq, sessions): trade_days = map(int, sessions.strftime("%Y%m%d")) if freq == '1m': freq = '1 min' if freq == '1d': freq = '24 H' res = [] concurrent_count = self.concurrent_thread_count jobs = [] for trade_day in trade_days: # df = Engine.minute_bars_from_transaction(self._get_transaction(code, trade_day), freq) reqevent = gevent.spawn(Engine.minute_bars_from_transaction, self._get_transaction(code, trade_day), freq) jobs.append(reqevent) if len(jobs) >= concurrent_count: gevent.joinall(jobs, timeout=30) for j in jobs: if j.value is not None and not j.value.empty: res.append(j.value) jobs.clear() gevent.joinall(jobs, timeout=30) for j in jobs: if j.value is not None and not j.value.empty: res.append(j.value) jobs.clear() if len(res) != 0: return pd.concat(res) return pd.DataFrame() def get_k_data(self, code, start, end, freq, check=True): if isinstance(start, str) or isinstance(end, str): start = pd.Timestamp(start) end = pd.Timestamp(end) if check: daily_bars = self.get_security_bars(code, '1d', start, end) if daily_bars is None or daily_bars.empty: return daily_bars sessions = daily_bars.index else: sessions = pd.bdate_range(start, end, weekmask='Mon Tue Wed Thu Fri') df = self._get_k_data(code, freq, sessions) def check_df(freq, df, daily_bars): if freq == '1m': need_check = pd.DataFrame({ 'open': df['open'].resample('1D').first(), 'high': df['high'].resample('1D').max(), 'low': df['low'].resample('1D').min(), 'close': df['close'].resample('1D').last(), 'volume': df['volume'].resample('1D').sum() }).dropna() else: need_check = df if daily_bars.shape[0] != need_check.shape[0]: logger.warning("{} merged {}, expected {}".format( code, need_check.shape[0], daily_bars.shape[0])) need_check = fillna( need_check.reindex(daily_bars.index, copy=False)) diff = daily_bars[['open', 'close']] == need_check[['open', 'close']] res = (diff.open) & (diff.close) sessions = res[res == False].index return sessions if not df.empty: if check: sessions = check_df(freq, df, daily_bars) if sessions.shape[0] != 0: logger.info( "fixing data for {}-{} with sessions: {}".format( code, freq, sessions)) fix = self._get_k_data(code, freq, sessions) df.loc[fix.index] = fix return df return df
def single_client_benchmark(ip): def _log(msg): click.echo("HQ_BENCHMARK: [{:15s}] {} ".format(ip, datetime.datetime.now()) + msg) def _grouped_list(stocks): return [stocks[i:i + GET_QUOTES_PER_GROUPS] for i in range(0, len(stocks), GET_QUOTES_PER_GROUPS)] _log("start benchmark") total_time = connecting_time = get_security_count_time = get_security_list_time = get_security_quotes_time = num = 0 start_time = time.time() last_time = start_time try: api = TdxHq_API(multithread=True) port = 7709 if ":" in ip: ip, port = ip.split(':') port = int(port) with api.connect(ip=ip, port=port): _log("connected") cur_time = time.time() connecting_time = cur_time - last_time last_time = cur_time _log("connecting time is {}".format(connecting_time)) num = api.get_security_count(0) _log("all shenzhen market stock count is {}".format(num)) cur_time = time.time() get_security_count_time = cur_time - last_time last_time = cur_time _log("get_security_count_time is {}".format(get_security_count_time)) all = [] for i in range((num // 1000) + 1): offset = i * 1000 section = api.get_security_list(0, offset) all = all + section cur_time = time.time() get_security_list_time = cur_time - last_time last_time = cur_time _log("get_security_list_time is {}".format(get_security_list_time)) codes = [one['code'] for one in all] results = [] for stocks in _grouped_list(codes): req_list = [(0, code) for code in stocks] one_results = api.get_security_quotes(req_list) results = results + one_results cur_time = time.time() get_security_quotes_time = cur_time - last_time last_time = cur_time _log("get_security_quotes_time is {}".format(get_security_quotes_time)) total_time = last_time - start_time _log("total_time is {}".format(total_time)) _log("end benchmark") except Exception as e: _log("hit exception " + str(e)) return { "ip": ip, "total_time": total_time, "connecting_time": connecting_time, "get_security_count_time": get_security_count_time, "get_security_list_time": get_security_list_time, "get_security_quotes_time": get_security_quotes_time, "security_count": num }
def single_client_benchmark(ip): def _log(msg): click.echo("HQ_BENCHMARK: [{:15s}] {} ".format(ip, datetime.datetime.now()) + msg) def _grouped_list(stocks): return [stocks[i:i + GET_QUOTES_PER_GROUPS] for i in range(0, len(stocks), GET_QUOTES_PER_GROUPS)] _log("start benchmark") total_time = connecting_time = get_security_count_time = get_security_list_time = get_security_quotes_time = num = 0 start_time = time.time() last_time = start_time try: api = TdxHq_API(multithread=True) with api.connect(ip=ip): _log("connected") cur_time = time.time() connecting_time = cur_time - last_time last_time = cur_time _log("connecting time is {}".format(connecting_time)) num = api.get_security_count(0) _log("all shenzhen market stock count is {}".format(num)) cur_time = time.time() get_security_count_time = cur_time - last_time last_time = cur_time _log("get_security_count_time is {}".format(get_security_count_time)) all = [] for i in range((num // 1000) + 1): offset = i * 1000 section = api.get_security_list(0, offset) all = all + section cur_time = time.time() get_security_list_time = cur_time - last_time last_time = cur_time _log("get_security_list_time is {}".format(get_security_list_time)) codes = [one['code'] for one in all] results = [] for stocks in _grouped_list(codes): req_list = [(0, code) for code in stocks] one_results = api.get_security_quotes(req_list) results = results + one_results cur_time = time.time() get_security_quotes_time = cur_time - last_time last_time = cur_time _log("get_security_quotes_time is {}".format(get_security_quotes_time)) total_time = last_time - start_time _log("total_time is {}".format(total_time)) _log("end benchmark") except Exception as e: _log("hit exception " + str(e)) return { "ip": ip, "total_time": total_time, "connecting_time": connecting_time, "get_security_count_time": get_security_count_time, "get_security_list_time": get_security_list_time, "get_security_quotes_time": get_security_quotes_time, "security_count": num }
class TdxHelper: ip_list = [{ 'ip': '119.147.212.81', 'port': 7709 }, { 'ip': '60.12.136.250', 'port': 7709 }] def __init__(self): #连接tdx接口 self.api = TdxHq_API() if not self.api.connect('60.12.136.250', 7709): print("服务器连接失败!") # pandas数据显示设置 pd.set_option('display.max_columns', None) # 显示所有列 #pd.set_option('display.max_rows', None) # 显示所有行 # mysql对象 self.mysql = mysqlHelper(config.mysql_host, config.mysql_username, bluedothe.mysql_password, config.mysql_dbname) # pandas的mysql对象 self.engine = create_engine( f'mysql+pymysql://{config.mysql_username}:{bluedothe.mysql_password}@{config.mysql_host}/{config.mysql_dbname}?charset=utf8' ) #断开tdx接口连接 def close_connect(self): self.api.disconnect() #获取k线,最后一个参数day,说明需要获取的数量,本接口只获取从最近交易日往前的数据 #输入参数:五个参数分别为:category(k线),市场代码(0:深圳,1:上海),股票代码,开始位置(从最近交易日向前取,0表示最近交易日),返回的记录条数 #K线种类: 0 5分钟K线; 1 15分钟K线; 2 30分钟K线; 3 1小时K线; 4 日K线;5 周K线;6 月K线;7 1分钟;8 1分钟K线; 9 日K线;10 季K线;11 年K线 #返回值:open,close,high,low,vol,amount,year,month,day,hour,minute,datetime # csv格式:code,ts_code,trade_date(缩写),trade_time,time_index,open,high,low,close,amount,volume def get_security_bars(self, category, market, code, start=0, count=240): dict = {0: 'SZ', 1: 'SH'} ts_code = code + "." + dict[market] order = [ 'code', 'ts_code', 'trade_date', 'trade_time', 'time_index', 'open', 'high', 'low', 'close', 'amount', 'volume' ] #df = self.api.get_security_bars(9, 0, '000001', 0, 10) # 返回普通list df = self.api.to_df( self.api.get_security_bars(category, market, code, start, count)) # 返回DataFrame if df.empty: return df df.insert(0, 'ts_code', ts_code) df.insert(0, 'code', code) df['trade_time'] = df['datetime'].apply(lambda x: str(x)[11:19]) df['time_index'] = df['trade_time'].apply( lambda x: datatime_util.stockTradeTime2Index(x)) df['trade_date'] = df['datetime'].apply( lambda x: (str(x)[0:10]).replace('-', '')) df.rename(columns={'vol': 'volume'}, inplace=True) df.drop(['year', 'month', 'day', 'hour', 'minute', 'datetime'], axis=1, inplace=True) df['volume'] = df['volume'].apply(lambda x: int(x)) #取整 df.loc[df['amount'] == 5.877471754111438e-39, 'amount'] = 0 #列值根据条件筛选后修改为0 df = df[order] filename = config.tdx_csv_minline1_all + ts_code + ".csv" if os.path.isfile(filename): df.to_csv(filename, index=False, mode='a', header=False, sep=',', encoding="utf_8_sig") else: df.to_csv(filename, index=False, mode='w', header=True, sep=',', encoding="utf_8_sig") print("新增加的一分钟all股票数据:", filename) # 获取1分钟k线,最后一个参数说明需要获取的数量,本接口只获取从最近交易日往前的数据 # 输入参数:五个参数分别为:category(k线),市场代码(0:深圳,1:上海),股票代码,开始位置(从最近交易日向前取,0表示最近交易日),返回的记录条数 # K线种类: 0 5分钟K线; 1 15分钟K线; 2 30分钟K线; 3 1小时K线; 4 日K线;5 周K线;6 月K线;7 1分钟;8 1分钟K线; 9 日K线;10 季K线;11 年K线 # 返回值:open,close,high,low,vol,amount,year,month,day,hour,minute,datetime # csv格式:code,ts_code,trade_date(缩写),trade_time,time_index,open,high,low,close,amount,volume def get_security_bars_minute1(self, category, market, code, start, count): dict = {0: 'SZ', 1: 'SH'} ts_code = code + "." + dict[market] order = [ 'code', 'ts_code', 'trade_date', 'trade_time', 'time_index', 'open', 'high', 'low', 'close', 'amount', 'volume' ] # df = self.api.get_security_bars(9, 0, '000001', 0, 10) # 返回普通list df = self.api.to_df( self.api.get_security_bars(category, market, code, start, count)) # 返回DataFrame if df.empty: return df.insert(0, 'ts_code', ts_code) df.insert(0, 'code', code) df['trade_time'] = df['datetime'].apply(lambda x: str(x)[11:19]) df['time_index'] = df['trade_time'].apply( lambda x: datatime_util.stockTradeTime2Index(x)) df['trade_date'] = df['datetime'].apply( lambda x: (str(x)[0:10]).replace('-', '')) df.rename(columns={'vol': 'volume'}, inplace=True) df.drop(['year', 'month', 'day', 'hour', 'minute', 'datetime'], axis=1, inplace=True) df['volume'] = df['volume'].apply(lambda x: int(x)) # 取整 df.loc[df['amount'] == 5.877471754111438e-39, 'amount'] = 0 # 列值根据条件筛选后修改为0 df = df[order] #过滤掉停牌的数据,在tdx中,停牌股票也能取到数据,价格是前一交易日的收盘价,所以只能用成交量或成交金额为0来判断 #1按日期分组后取出成交量为0的日期;2循环过滤掉成交量为0的日期的数据。 dfg = df.groupby(by='trade_date').mean() #分组 dfg['trade_date'] = dfg.index dfg = dfg[dfg.volume == 0] #条件过滤,保留满足条件的数据 for trade_date in dfg['trade_date'].values: df = df[(df['trade_date'] != trade_date)] # 每个条件要用括号()括起来 return df #可以获取多只股票的行情信息 #返回值:market,code,active1,price,last_close,open,high,low,reversed_bytes0,reversed_bytes1,vol,cur_vol,amount,s_vol, #reversed_bytes2,reversed_bytes3,bid1,ask1,bid_vol1,ask_vol1,bid2,ask2,bid_vol2,ask_vol2,bid3,ask3,bid_vol3,ask_vol3,bid4, #ask4,bid_vol4,ask_vol4,bid5,ask5,bid_vol5,ask_vol5,reversed_bytes4,reversed_bytes5,reversed_bytes6,reversed_bytes7, #reversed_bytes8,reversed_bytes9,active2 def get_security_quotes(self): df = self.api.to_df( self.api.get_security_quotes([(0, '000001'), (1, '600300')])) print(df) # 获取市场股票数量 #返回值:value def get_security_count(self): df = self.api.to_df(self.api.get_security_count(0)) #0 - 深圳, 1 - 上海 print(df) # 获取股票列表,返回值里面除了股票,还有国债等 #返回值:code,volunit,decimal_point,name,pre_close def get_security_list(self): df = self.api.to_df(self.api.get_security_list( 0, 10000)) # 市场代码, 起始位置 如: 0,0 或 1,100 print(df) # 获取指数k线 #输入参数同股票k线接口 # 返回值:open,close,high,low,vol,amount,year,month,day,hour,minute,datetime,up_count down_count def get_index_bars(self): index_dict_cn = { "上证指数": "999999", "深证成指": "399001", "中小板指": "399005", "创业板指": "399006", "深证综指": "399106", "上证50": "000016", "沪深300": "000300" } index_dict = { "sh": "999999", "sz": "399001", "zxb": "399005", "cyb": "399006", "szz": "399106", "sz50": "000016", "hs300": "000300" } for key in index_dict.keys(): df = self.api.to_df( self.api.get_index_bars(9, 1, index_dict[key], 0, 2)) print(df) # 查询分时行情,最近交易日的数据,一分钟一条记录 #返回值:price,vol def get_minute_time_data(self): df = self.api.to_df(self.api.get_minute_time_data( 1, '600300')) #市场代码, 股票代码 print(df) # 查询历史分时行情 # 返回值:price,vol def get_history_minute_time_data(self): df = self.api.to_df( self.api.get_history_minute_time_data(TDXParams.MARKET_SH, '603887', 20200420)) #市场代码, 股票代码,时间 print(df) # 查询分笔成交,最近交易日数据 #返回值:time,price,vol,num,buyorsell def get_transaction_data(self): df = self.api.to_df( self.api.get_transaction_data(TDXParams.MARKET_SZ, '000001', 0, 30)) #市场代码, 股票代码,起始位置, 数量 print(df) # 查询历史分笔成交 #返回值:time,price,vol,buyorsell def get_history_transaction_data(self): df = self.api.to_df( self.api.get_history_transaction_data( TDXParams.MARKET_SZ, '000001', 0, 10, 20170209)) #市场代码, 股票代码,起始位置,日期 数量 print(df) # 查询公司信息目录,返回的不是具体数据 #返回值:name,filename,start,length def get_company_info_category(self): df = self.api.to_df( self.api.get_company_info_category(TDXParams.MARKET_SZ, '000001')) #市场代码, 股票代码 print(df) # 读取公司信息详情 #返回值:value def get_company_info_content(self): df = self.api.to_df( self.api.get_company_info_content( 0, '000001', '000001.txt', 0, 1000)) #市场代码, 股票代码, 文件名, 起始位置, 数量 print(df) # 读取除权除息信息 #返回值:year,month,day,category,name,fenhong,peigujia,songzhuangu,peigu def get_xdxr_info(self): df = self.api.to_df(self.api.get_xdxr_info(1, '600300')) #市场代码, 股票代码 print(df) # 读取财务信息 #返回值:market,code,liutongguben,province,industry,updated_date,ipo_date,zongguben,guojiagu,faqirenfarengu,farengu,bgu,hgu,zhigonggu, #zongzichan,liudongzichan,gudingzichan,wuxingzichan,gudongrenshu,liudongfuzhai,changqifuzhai,zibengongjijin,jingzichan,zhuyingshouru, #zhuyinglirun,yingshouzhangkuan,yingyelirun,touzishouyu,jingyingxianjinliu,zongxianjinliu,cunhuo,lirunzonghe,shuihoulirun,jinglirun,weifenlirun,baoliu1,baoliu2 def get_finance_info(self): df = self.api.to_df(self.api.get_finance_info(1, '600300')) #市场代码, 股票代码 print(df) # 读取k线信息 # 返回值:value def get_k_data(self): df = self.api.to_df( self.api.get_k_data('600300', '2017-07-03', '2017-07-10')) #股票代码, 开始时间, 结束时间 print(df) # 读取板块信息 #返回值:blockname, block_type, code_index, code """ BLOCK_SZ = "block_zs.dat";BLOCK_FG = "block_fg.dat";BLOCK_GN = "block_gn.dat";BLOCK_DEFAULT = "block.dat" """ def get_and_parse_block_info(self): ##指数板块 风格板块 概念板块 一般板块 block_filename = [ "block_zs.dat", "block_fg.dat", "block_gn.dat", "block.dat" ] for block in block_filename: df = self.api.to_df( self.api.get_and_parse_block_info(block)) #板块文件名称 filename = config.tdx_csv_block + block[0:-4] + ".csv" if os.path.isfile(filename): os.remove(filename) df.to_csv(filename, index=False, mode='w', header=True, sep=',', encoding="utf_8_sig") else: df.to_csv(filename, index=False, mode='w', header=True, sep=',', encoding="utf_8_sig") # 读取板块信息,多个类型封装到一个df对象中返回 # 返回值:data_source, block_category, block_type, block_name, block_code, ts_code, create_time def update_block_member(self): ##指数板块 风格板块 概念板块 一般板块 #block_filename = ["block_zs.dat", "block_fg.dat", "block_gn.dat", "block.dat"] block_filename = ["block_zs.dat", "block_fg.dat", "block_gn.dat"] #block.dat中的数据都包含在其他版块里了,这个可以去掉 data_source = "tdx" dfall = None for block in block_filename: df = self.api.to_df( self.api.get_and_parse_block_info(block)) # 板块文件名称 df['data_source'] = data_source if block == "block.dat": df['block_category'] = data_source + ".yb" else: df['block_category'] = data_source + "." + block[6:8] df['block_type'] = df['block_type'].map(lambda x: str(x)) df['block_type'] = df['block_category'].str.cat( df['block_type'], sep=".") #, sep = "." df['block_code'] = "" #使用pd直接插入到数据库时,字段不能是None值 df['ts_code'] = df['code'].apply(lambda x: x + ".SH" if x[0:1] == "6" else x + ".SZ") if (dfall is not None) and (not dfall.empty): dfall = dfall.append(df, ignore_index=True) else: dfall = df if (dfall is None) or (dfall.empty): return None dfall.rename(columns={'blockname': 'block_name'}, inplace=True) dfall['create_time'] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) dfall = dfall[[ 'data_source', 'block_category', 'block_type', 'block_name', 'block_code', 'ts_code', 'create_time' ]] #列重排序 #分组统计 dfg = dfall.groupby(by=[ 'data_source', 'block_category', 'block_type', 'block_name', 'block_code' ], as_index=False).count() # 分组求每组数量 dfg.rename(columns={'ts_code': 'member_count'}, inplace=True) #ts_code列数值为汇总值,需要重命名 dfg['create_time'] = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time())) #create_time列数值为汇总值,需要重新赋值 delete_condition = f"data_source = '{data_source}'" mysql_script.df2db_update(delete_condition=delete_condition, block_basic_df=dfg, block_member_df=dfall) return (len(dfg), len(dfall)) #获取一段时间的1分钟数据,因为每次调用接口只能返回3天的分钟数据(240*3),需要分多次调用 #返回值:0没有提取到数据;1提取到数据 def get_minute1_data(self, category, market, code, start_date, end_date): init_start_date = start_date.replace('-', '') init_end_date = end_date.replace('-', '') day = datatime_util.diffrentPeriod(datatime_util.DAILY, start_date, end_date) df = self.get_security_bars_minute1(category, market, code, 0, 240 * 3) # 返回DataFrame if df is None or df.empty: print('{0}没有交易数据'.format(code)) return 0 print(market, '--', code, '--', start_date, '--', end_date) #print("最大值:",df.groupby('datetime').max()) #print(df.describe()) #df数据统计 data_start_date = df.min()['trade_date'] data_end_date = df.max()['trade_date'] start_date = start_date.replace('-', '') end_date = end_date.replace('-', '') if data_end_date < start_date or end_date < data_start_date: print("采集时间在数据范围之外,退出函数") return 0 elif end_date > data_end_date: end_date = data_end_date if start_date < data_start_date: #最近三天的数据中,去掉无用的数据后即是最终数据 #需要取的数据还有三天前的数据,需要继续向前取 n = (day - 3) // 3 m = (day - 3) % 3 for i in range(0, n): dfn = self.get_security_bars_minute1(category, market, code, 240 * 3 * (i + 1), 240 * 3) # 返回DataFrame if (dfn is not None) and (not dfn.empty): df = dfn.append(df, ignore_index=True) if m > 0: dfn = self.get_security_bars_minute1(category, market, code, 240 * 3 * (n + 1), 240 * m) if (dfn is not None) and (not dfn.empty): df = dfn.append(df, ignore_index=True) df = df.sort_values(by=['trade_date', 'time_index'], axis=0, ascending=True) #过滤掉start_date, end_date之外的数据 df = df[(df['trade_date'] >= str(init_start_date)) & (df['trade_date'] <= str(init_end_date))] #每个条件要用括号()括起来 dict = {0: 'SZ', 1: 'SH'} ts_code = code + "." + dict[market] filename = config.tdx_csv_minline1_all + ts_code + ".csv" if os.path.isfile(filename): df.to_csv(filename, index=False, mode='a', header=False, sep=',', encoding="utf_8_sig") print("更新一分钟all股票数据:", filename) else: df.to_csv(filename, index=False, mode='w', header=True, sep=',', encoding="utf_8_sig") print("新增加的一分钟all股票数据:", filename)
class TDX(object): ''' This class is tong da xin data source. We can use it to get down the stock datas. Tushare can't get minter line and or year line. TDX can search index of stock and funds. ''' def __init__(self): self.tdx_api = TdxHq_API() self.__ip = '119.147.212.81' #输入IP self.__port = 7709 #端口 self.__code = '600200' self.__market = 1 #市场代码 0:深圳,1:上海 self._startdate = "2017-01-01" self.today = datetime.date.today() self._enddate = datetime.datetime.strftime(self.today, '%Y-%m-%d') self.__mkt_segment = { 'sh': '60', "sz": '00', "cyb": "30", } #segment 当前板块开始字符串 def __str__(self): return 'TDX object (code : %s)' % self.code @property def IP(self): # self.IP return self.__ip @property def PORT(self): return self.__port @property def code(self): #定义stock code 属性 return self.__code @code.setter #设定code def code(self, code_input): """ The setter of the code property """ if not isinstance(code_input, str): #确定是否是字符串 raise ValueError('the code must string!') if not len(code_input) == 6: #确定长度 raise ValueError('the code value error,the len must SIX !') if code_input.startswith('60'): #确定表头 self.__market = 1 elif code_input.startswith('00'): self.__market = 0 elif code_input.startswith('30'): self.__market = 0 else: raise ValueError('this code is not stock code') self.__code = code_input @property def startdate(self): #开始日期 return self._startdate @startdate.setter #设置日期 def startdate(self, date_input): """ The setter of the start date property """ if not isinstance(date_input, str): raise ValueError('the date must string!') if not len(date_input) == 8: raise ValueError( 'the date value error,the date formet must xxxx-xx-xx !') self._startdate = date_input @property #结束日期 def enddate(self): return self._enddate @enddate.setter def enddate(self, date_input): """ The setter of the start date property """ if not isinstance(date_input, str): raise ValueError('the date must string!') if not len(date_input) == 8: raise ValueError( 'the date value error,the date formet must xxxx-xx-xx !') self._enddate = date_input def get_day_data_tdx(self): #获取K line with self.tdx_api.connect(self.IP, self.PORT): data = self.tdx_api.get_k_data(self.code, self.startdate, self.enddate) data = pandas.DataFrame(data) data.date = data.date.apply( lambda x: datetime.datetime.strptime(x, "%Y-%m-%d")) return data #TODO: 现在是用800点进行计数,以后会细化功能 def get_k_data_tdx(self, k_mode=9): """ 获取k 线图,总计800 点 Parameters ---------- k_mode= 0-11 0 5分钟K线 1 15分钟K线 2 30分钟K线 3 1小时K线 4 日K线 5 周K线 6 月K线 7 1分钟 8 1分钟K线 9 日K线 10 季K线 11 年K线 Returns ------- """ with self.tdx_api.connect(self.self.IP, self.self.PORT): data = self.tdx_api.get_security_bars(k_mode, self.__market, self.code, 0, 800) data = pandas.DataFrame(data) #data.date = data.date.apply( # lambda x: datetime.datetime.strptime(x, "%Y-%m-%d")) return data def len_market(self): #市场有多少只股票 with self.tdx_api.connect(self.IP, self.PORT): _len = self.tdx_api.get_security_count(self.__market) return _len def get_page_tdx(self, block=None): if block is None: market = 1 page = [0] elif block in ['sh', 'SH']: market = 1 page = [13, 14] elif block in ['sz', 'SZ']: print('block for shenzhen') market = 0 page = [0, 1] elif block in ['cyb', 'CYB']: print('block for chuang ye ban') market = 0 page = [7, 8] else: pass code_list_df = pandas.DataFrame() with self.tdx_api.connect(self.IP, self.PORT): for pn in page: data = self.tdx_api.get_security_list(market, pn * 1000) data = pandas.DataFrame(data) print(data) code_list_df = code_list_df.append(data, ignore_index=True) return code_list_df def get_base_finace_tdx(self): with self.tdx_api.connect(self.IP, self.PORT): data = self.tdx_api.get_finance_info(0, '000001') data = pandas.Series(data) print(data) def get_min_data(self): from pytdx.params import TDXParams with self.tdx_api.connect(self.IP, self.PORT): data = self.tdx_api.get_history_minute_time_data( TDXParams.MARKET_SH, self.code, 20161209) data = pandas.DataFrame(data) print(data) #TODO: 需要确定 0: buy 1 : sell def get_tick_data(self): """ 历史分笔交易:time 顺序; price ; vol ;buyorsell [1:] [0:]; sh 60 13000-14000 Parameters ---------- Returns ------- """ data = pandas.DataFrame() with self.tdx_api.connect(self.IP, self.PORT): for i in [2000, 0000]: df = self.tdx_api.get_history_transaction_data( TDXParams.MARKET_SH, "600547", i, 2000, 20160308) df = pandas.DataFrame(df) data = data.append(df, ignore_index=True) return data def get_tick_today(self): """ Get every time the each deal for today.每组数最大len 2 k 所以要确定的数据长度 Parameters ---------- self: Returns ------- """ with self.tdx_api.connect(self.IP, self.PORT): data = pandas.DataFrame() for i in [0, 2000]: df = self.tdx_api.get_transaction_data(self.__market, self.code, i, 2000) df = pandas.DataFrame(df) data = data.append(df, ignore_index=True) return data def get_block(self): with self.tdx_api.connect(self.IP, self.PORT): data = self.tdx_api.get_and_parse_block_info("block.dat") data = pandas.DataFrame(data) print(data) def get_market_segment_list(self, mkt): data = self.get_page_tdx(mkt) self.code_list = pandas.DataFrame() pbar = tqdm(total=len(data.code)) mkt_hard = self.mkt_segment[mkt] for idx, __code in enumerate(data.code): pbar.update(1) if __code.startswith(mkt_hard, 0, 2): self.code_list = self.code_list.append(data.loc[idx], ignore_index=True) return self.code_list def get_sh_list(self): return self.get_market_segment_list('sh') def get_sz_list(self): return self.get_market_segment_list('sz') def get_cyb_list(self): return self.get_market_segment_list('cyb')
class TDX: HQ_HOSTS_FILE = os.path.join(os.getcwd(), "tdx_hq_hosts.json") def __init__(self, root): self.root = root self.tdx = TdxHq_API(heartbeat=True, auto_retry=True) self.connect_to_server() @staticmethod async def ping(host): timeout = 999999 try: delay = await aioping.ping(host[1]) * 1000 print( 'ping host[' + host[0] + ', ' + str(host[1]) + ':' + str(host[2]) + '] ... ... ', round(delay, 2), 'ms') except TimeoutError: delay = timeout print('ping host[' + host[0] + ', ' + str(host[1]) + ':' + str(host[2]) + '] ... ... timeout') return delay @staticmethod async def test_hosts(hosts): return await asyncio.gather(*[TDX.ping(host) for host in hosts]) @staticmethod def find_available_hosts(hq_hosts=tdx_hq_hosts): thread = ThreadedAsyncio(target=TDX.test_hosts, args=(hq_hosts, )) thread.start() results = thread.join() hosts = [] for idx, host in enumerate(hq_hosts): if results[idx] > 500: continue host = [*host] if len(host) == 3: host.append(results[idx]) else: host[-1] = results[idx] hosts.append(host) hosts.sort(key=lambda x: x[3]) with open(TDX.HQ_HOSTS_FILE, "w") as f: f.write(json.dumps(hosts)) return hosts def connect_to_server(self): if not os.path.exists(TDX.HQ_HOSTS_FILE): print('use default server: ', '202.108.253.130:7709') print( 'please run TDX.find_available_hosts to test out the fastest servers' ) self.tdx.connect('202.108.253.130', 7709) else: with open(TDX.HQ_HOSTS_FILE) as f: hosts = json.load(f) hosts = TDX.find_available_hosts(hq_hosts=hosts) for host in hosts: try: print('try to connect the fastest host:', host) self.tdx.connect(host[1], host[2]) break except Exception as e: print(str(e)) print('retry with next host') def get_tdx_gainian(self): fname = os.path.join(self.root, 'T0002', 'hq_cache', 'block_gn.dat') result = {} if type(fname) is not bytearray: with open(fname, "rb") as f: data = f.read() else: data = fname pos = 384 (num, ) = struct.unpack("<H", data[pos:pos + 2]) pos += 2 for i in range(num): blockname_raw = data[pos:pos + 9] pos += 9 name = blockname_raw.decode("gbk", 'ignore').rstrip("\x00") stock_count, block_type = struct.unpack("<HH", data[pos:pos + 4]) pos += 4 block_stock_begin = pos codes = [] for code_index in range(stock_count): one_code = data[pos:pos + 7].decode("utf-8", 'ignore').rstrip("\x00") codes.append(one_code) pos += 7 gn = {} gn["name"] = name gn["block_type"] = block_type gn["stock_count"] = stock_count gn["codes"] = codes result[name] = gn pos = block_stock_begin + 2800 return result def get_tdx_hangye(self): file_hangye = os.path.join(self.root, 'incon.dat') assert os.path.exists(file_hangye) file_stock_hangye = os.path.join(self.tdx_dir, 'T0002', 'hq_cache', ' tdxhy.cfg') assert os.path.exists(file_stock_hangye) result = {} with open(file_hangye, "rt", encoding='gb2312') as f: isTDXHY = False for line in f: line = line.rstrip() if not isTDXHY and line != '#TDXNHY': continue elif not isTDXHY and line == '#TDXNHY': isTDXHY = True continue elif isTDXHY and line == '######': isTDXHY = False break code, name = line.split('|') result[code] = {} result[code]['code'] = code result[code]['name'] = name result[code]['codes'] = [] with open(file_stock_hangye, "rt", encoding='gb2312') as f: for line in f: line = line.rstrip() market_code, stock_code, tdxhy_code, swhy_code, unknown_code = line.split( "|") stock_code = stock_code.strip() if tdxhy_code != 'T00': result[tdxhy_code]['codes'].append(stock_code) return result def get_tdx_zhishu(self): tdxzs_cfg = os.path.join(self.root, 'T0002', 'hq_cache', 'tdxzs.cfg') gainian = self.get_tdx_gainian() hangye = self.get_tdx_hangye() result = {} with open(tdxzs_cfg, "rt", encoding='gb2312') as f: for line in f: line = line.rstrip() zs_name, zs_code, zs_type, num_1, num_2, key = line.split('|') if key in gainian: if zs_code in result: print( '------------------------------------------------------' ) print('in result key: ', key, zs_name, zs_code) print('gainian: ', key, gainian[key]) continue else: if len(gainian[key]['codes']) == 0: continue zs = {} zs['code'] = zs_code zs['name'] = gainian[key]['name'] zs['codes'] = gainian[key]['codes'] result[zs_code] = zs if key in hangye: if zs_code in result: print( '------------------------------------------------------' ) print('in result key: ', key, zs_name, zs_code) print('hangye: ', key, hangye[key]) continue else: if len(hangye[key]['codes']) == 0: continue zs = {} zs['code'] = zs_code zs['name'] = hangye[key]['name'] zs['codes'] = hangye[key]['codes'] result[zs_code] = zs return result def is_tdx_local_data_ready_for(self, dt): file = os.path.join(self.root, 'vipdoc', 'sz', 'lday', 'sz399001.day') reader = TdxDailyBarReader() df = reader.get_df(file) return dt.strftime('%Y-%m-%d') in df.index def get_lastest_stock_codes(self): old_codes = Utils.get_stock_codes() code_filter = Utils.code_filter step_size = 1000 codes = [] for market in Utils.markets: count = self.tdx.get_security_count(market) print(market, count, end=' : ') steps = math.ceil(count / step_size) total = 0 for step in range(steps): result = self.tdx.get_security_list(market, step_size * step) print(str(step) + '/' + str(steps), end=", ") for item in result: code = item['code'].strip() if code[:3] in code_filter[market]: codes.append(code) total += 1 market_name = '深市' if market == 0 else '沪市' print(market_name + ' A股 总数: ' + str(total)) print('沪深 A股 总数:' + str(len(codes))) return codes