def worker(q, hy_map, hy_code): index_market, index_code = hy_map[hy_code]["market"], hy_map[hy_code][ "code"] stock_lst = hy_map[hy_code]["stock"] download_lst = [dict(market=index_market, code=index_code, index=True)] download_lst.extend(stock_lst) # 下载指数 historical data try: server_addr = q.get(timeout=2) except Exception as e: print("没有可用的服务器") return api = TdxHq_API(heartbeat=True, auto_retry=True, multithread=True) try: if api.connect(*server_addr[1:]): for item in download_lst: market = item["market"] code = item["code"] isindex = item.get("index", False) download(api, market, code, years, isindex=isindex) else: print("连接失败") except Exception as e: print(e) traceback.print_exc() print("进程下载失败") finally: api.disconnect() q.put(server_addr) return
def __hq_ping(self, ip, port): api = TdxHq_API() with api.connect(ip, port, time_out=0.7): assert len(api.get_security_list(0, 1)) > 800 api.disconnect() return True return False
def main(): manager = Manager() q = manager.Queue() if not path.exists(stock_data_path): os.makedirs(stock_data_path) if not path.exists(index_data_path): os.makedirs(index_data_path) hy_map = get_hy(tdxzs_path, tdxhy_path) print("总共行业数量: %s" % len(hy_map)) pool = ProcessPoolExecutor(process_pool_size) api = TdxHq_API(heartbeat=True, auto_retry=True, multithread=True) print("寻找可用服务器...") server_count = 0 for data_server in random.choices(hq_hosts, k=4): try: if api.connect(*data_server[1:]): q.put(data_server) server_count += 1 except Exception: print("连接服务器失败") finally: api.disconnect() print("可用服务器: %s" % server_count) if server_count < 1: print("没有可用服务器") print("开始下载数据...") fut_lst = [] for hy_code in hy_map: fut_lst.append(pool.submit(worker, q, hy_map, hy_code)) # print(len(fut_lst)) wait(fut_lst) print(fut_lst[0].result())
def get_a_quote_by_tdx(code_list): """ 获取A股市场行情,该接口只获取A股市场行情 其他的行情使用拓展行情接口 code_list:数字或者字符串格式的list,调用def standard_tdx_code(code_list)来格式化市场信息及code """ if len(code_list) == 0: return pd.DataFrame(columns=stock_tdx_columns) # 一次最多获取50只股票的实时行情,如果code_list 多于50只,则50只股票取一次行情,然后进行拼接 tdx_api = TdxHq_API() if len(code_list) >= 50: tmp = div_list(code_list, 50) df_tmp = pd.DataFrame(columns=stock_tdx_columns) if tdx_api.connect('119.147.212.81', 7709): for tmp_list in tmp: tmp_list = standard_tdx_code(tmp_list) tdx_data = tdx_api.to_df(tdx_api.get_security_quotes(tmp_list)) df_tmp = pd.concat([df_tmp, tdx_data]) df_tmp = df_tmp[stock_tdx_columns] else: tmp = standard_tdx_code(code_list) if tdx_api.connect('119.147.212.81', 7709): tdx_data = tdx_api.to_df(tdx_api.get_security_quotes(tmp)) df_tmp = tdx_data[stock_tdx_columns] tdx_api.disconnect() return process_tdx_price(df_tmp)
def get_stock_count(): api = TdxHq_API() if api.connect('119.147.212.81', 7709): print(api.get_security_count(0)) #参数为市场代码 print(api.get_security_list(0, 0)) #第一个参数为市场代码,第二个参数为起始位置 print(api.get_security_count(1)) print(api.to_df(api.get_security_list(1, 0))) api.disconnect()
def _getKLineData(self): api = TdxHq_API(auto_retry=True) if api.connect('119.147.212.81', 7709): self._KLineData = api.to_df( api.get_index_bars(9, 1, self._stockCode, 0, self._stockOberCount)) # 返回DataFrame # print(self._KLineData) api.disconnect()
def get_company_info(): api = TdxHq_API() if api.connect('119.147.212.81', 7709): print(api.get_company_info_category(TDXParams.MARKET_SZ, '000001')) #查询公司信息目录,参数:市场代码, 股票代码 api.get_company_info_content(0, '000001', os.getcwd() + "/tdx_file/" + '000001.txt', 0, 100) #读取公司信息详情,参数文件路径不知干什么 print(api.get_finance_info(0, '000001')) #读取财务信息 api.disconnect()
def __hq_tick(self, code): api = TdxHq_API() ip, port = self.__hq.get() with api.connect(ip, port): df = api.get_security_quotes(code) if df is not None: df = api.to_df(df) api.disconnect() self.__hq.put((ip, port)) return df
def get_block_info(): api = TdxHq_API() if api.connect('119.147.212.81', 7709): data = api.get_and_parse_block_info("block.dat") #一般板块 #print(api.get_and_parse_block_info("block_zs.dat")) #指数板块 #print(api.get_and_parse_block_info("block_fg.dat")) #风格板块 #print(api.get_and_parse_block_info("block_gn.dat")) #概念板块 datadf = api.to_df(data) print(datadf) api.disconnect()
def get_kline_data(): api = TdxHq_API() if api.connect('119.147.212.81', 7709): data = api.get_security_bars( 9, 0, '000001', 0, 10) # 返回普通list,五个参数分别为:category(k线),市场代码(深市),股票代码,开始时间,记录条数 print(data) data = api.to_df(api.get_security_bars(9, 0, '000001', 0, 10)) # 返回DataFrame print(data) api.disconnect()
def quote_TDX(ds_tuple): api = TdxHq_API() if api.connect('119.147.212.81', 7709): MainCodeData = api.get_security_quotes([(1, code)]) n = ceil(len(ds_tuple) / 80) for i in range(n): if i == 0: quot = api.get_security_quotes(ds_tuple[i * 80:(i + 1) * 80]) #每次最多请求80个数据? else: quot_add = api.get_security_quotes( ds_tuple[i * 80:(i + 1) * 80]) quot = quot + quot_add api.disconnect() return quot, MainCodeData
def __hq_list(self, market): assert self.__hq.qsize() > 0 api = TdxHq_API() ip, port = self.__hq.get() with api.connect(ip, port): df = list() for start in range(0, api.get_security_count(market=market), 1000): df.append(api.to_df(api.get_security_list(market, start))) api.disconnect() self.__hq.put((ip, port)) df = pandas.concat(df, sort=False).assign(sse=market) df = df[[ 'code', 'volunit', 'decimal_point', 'name', 'pre_close', 'sse' ]].dropna() df = df.assign(sse='sh' if market == CODE_MARKET_SH else 'sz', sec=get_code_type(df.code.tolist(), market)) return df return None
class SP(object): def __init__(self,userid="account4",server="http://192.168.118.1:5000"): self.server = server self.userid = userid self.api = TdxHq_API(heartbeat=True) self.trader = None self.trading = False self.TDX_IP_SETS = STOCK_IP_SETS def connect(self): for ip in self.TDX_IP_SETS: try: if self.api.connect(ip, 7709): return except: pass def disconnect(self): logger.info("[DISCONNECT]:start disconnect !!!!!") self.api.disconnect() logger.info("[DISCONNECT]:disconnect finished !!!!!") def initial(self): '''每天初始化设置 ''' logger.info("[INITIAL]:start initial !!!!!") logger.info("[INITIAL]:try to create connect... ") self.connect() self.trader = trade(UserID=self.userid,api=self.api,mock=False,server=self.server) logger.info("[INITIAL]:connect successful!") self.trading = self.judgetradeday() logger.info("[INITIAL]:initial finished !!!!!") def judgetradeday(self): today = datetime.datetime.today().date().strftime('%Y-%m-%d') df = ts.trade_cal() return df[(df["calendarDate"]==today)].isOpen.values[0] def run(self): if not self.trading:return logger.info("[RUN]:start run !!!!!") self.trader.autobuy() logger.info("[RUN]:run finished !!!!!")
class PYTDXService(): """pytdx数据服务类""" def __init__(self): """Constructor""" self.connected = False # 数据服务连接状态 self.hq_api = None # 行情API def connect_api(self): """连接API""" # 连接增强行情API并检查连接情况 try: if not self.connected: host = SETTINGS["TDX_HOST"] port = SETTINGS["TDX_PORT"] self.hq_api = TdxHq_API() self.hq_api.connect(host, port) self.connected = True return True except Exception: raise ConnectionError("pytdx连接错误") def get_realtime_data(self, symbol: str): """获取股票实时数据""" try: symbols = self.generate_symbols(symbol) df = self.hq_api.to_df(self.hq_api.get_security_quotes(symbols)) return df except Exception: raise ValueError("股票数据获取失败") @staticmethod def generate_symbols(symbol: str): """组装symbols数据,pytdx接收的是以市场代码和标的代码组成的元祖的list""" new_symbols = [] code, exchange = symbol.split('.') new_symbols.append((exchange_map[exchange], code)) return new_symbols def close(self): """数据服务关闭""" self.connected = False self.hq_api.disconnect()
class tdxApi: def __init__(self): self.ipset = [(v[1], v[2]) for v in hq_hosts] random.shuffle(self.ipset) self.ippool = AvailableIPPool(TdxHq_API, self.ipset) self.primary_ip, _ = self.ippool.sync_get_top_n(2) self.api = TdxHq_API(multithread=True, heartbeat=True, auto_retry=True) def connect(self): self.api.connect(self.primary_ip[0], self.primary_ip[1]) def release(self): self.api.disconnect() def getQuotes(self, stock_list): datas = self.api.get_security_quotes(stock_list) if datas is None: return None else: return self.api.to_df(datas) def getList(self, market, index): df = self.api.to_df(self.api.get_security_list(market, index)) return df def getCount(self, market=0): return self.api.get_security_count(market) def getMinQuotes(self, market, stock_code): datas = self.api.get_minute_time_data(market, stock_code) if datas is None: return None return self.api.to_df(datas) def getMinQuotesHis(self, market, stock_code, date): datas = self.api.get_history_minute_time_data(market, stock_code, date) if datas is None: return None return self.api.to_df(datas)
class SP(object): def __init__(self, userid="account4", server="http://192.168.118.1:5000"): self.server = server self.userid = userid self.api = TdxHq_API(heartbeat=True) self.trader = None self.trading = False self.TDX_IP_SETS = STOCK_IP_SETS def connect(self): for ip in self.TDX_IP_SETS: try: if self.api.connect(ip, 7709): return except: pass def disconnect(self): logger.info("[DISCONNECT]:start disconnect !!!!!") self.api.disconnect() logger.info("[DISCONNECT]:disconnect finished !!!!!") def initial(self): '''每天初始化设置 ''' logger.info("[INITIAL]:start initial !!!!!") logger.info("[INITIAL]:try to create connect... ") self.connect() self.trader = trade(UserID=self.userid, api=self.api, mock=False, server=self.server) logger.info("[INITIAL]:connect successful!") logger.info("[INITIAL]:initial finished !!!!!") def run(self): logger.info("[RUN]:start run !!!!!") self.trader.autobuy() logger.info("[RUN]:run finished !!!!!")
def __hq_bars(self, code, offset, frequency=9, index=False): assert self.__hq.qsize() > 0 api = TdxHq_API() if index is True: market_func = get_index_market bars_func = api.get_index_bars else: market_func = get_code_market bars_func = api.get_security_bars ip, port = self.__hq.get() with api.connect(ip, port): df = list() for _code in code: market = market_func(_code) for _start, _count in offset: df.append( api.to_df( bars_func(frequency, market, _code, _start, _count)).assign(code=_code)) api.disconnect() self.__hq.put((ip, port)) if len(df) < 1: return None return pandas.concat(df, sort=False)
class Engine: def __init__(self, *args, **kwargs): if kwargs.pop('best_ip', False): self.ip = self.best_ip else: self.ip = '14.17.75.71' self.ip = kwargs.pop('ip', '14.17.75.71') self.thread_num = kwargs.pop('thread_num', 1) if not PY2 and self.thread_num != 1: self.use_concurrent = True else: self.use_concurrent = False self.api = TdxHq_API(args, kwargs) if self.use_concurrent: self.apis = [ TdxHq_API(args, kwargs) for i in range(self.thread_num) ] self.executor = ThreadPoolExecutor(self.thread_num) def connect(self): self.api.connect(self.ip) if self.use_concurrent: for api in self.apis: api.connect(self.ip) return self def __enter__(self): return self def exit(self): self.api.disconnect() if self.use_concurrent: for api in self.apis: api.disconnect() def __exit__(self, exc_type, exc_val, exc_tb): self.api.disconnect() if self.use_concurrent: for api in self.apis: api.disconnect() def quotes(self, code): code = [code] if not isinstance(code, list) else code code = self.security_list[self.security_list.code.isin( code)].index.tolist() data = [ self.api.to_df( self.api.get_security_quotes(code[80 * pos:80 * (pos + 1)])) for pos in range(int(len(code) / 80) + 1) ] return pd.concat(data) # data = data[['code', 'open', 'high', 'low', 'price']] # data['datetime'] = datetime.datetime.now() # return data.set_index('code', drop=False, inplace=False) def stock_quotes(self): code = self.stock_list.index.tolist() if self.use_concurrent: res = { self.executor.submit(self.apis[pos % self.thread_num].get_security_quotes, code[80 * pos:80 * (pos + 1)]) \ for pos in range(int(len(code) / 80) + 1)} return pd.concat([self.api.to_df(dic.result()) for dic in res]) else: data = [ self.api.to_df( self.api.get_security_quotes(code[80 * pos:80 * (pos + 1)])) for pos in range(int(len(code) / 80) + 1) ] return pd.concat(data) @lazyval def security_list(self): return pd.concat([ pd.concat([ self.api.to_df(self.api.get_security_list( j, i * 1000)).assign(sse=0 if j == 0 else 1).set_index( ['sse', 'code'], drop=False) for i in range(int(self.api.get_security_count(j) / 1000) + 1) ], axis=0) for j in range(2) ], axis=0) @lazyval def stock_list(self): aa = map(stock_filter, self.security_list.index.tolist()) return self.security_list[list(aa)] @lazyval def best_ip(self): return select_best_ip() @lazyval def concept(self): return self.api.to_df( self.api.get_and_parse_block_info(TDXParams.BLOCK_GN)) @lazyval def index(self): return self.api.to_df( self.api.get_and_parse_block_info(TDXParams.BLOCK_SZ)) @lazyval def fengge(self): return self.api.to_df( self.api.get_and_parse_block_info(TDXParams.BLOCK_FG)) @lazyval def customer_block(self): return CustomerBlockReader().get_df(CUSTOMER_BLOCK_PATH) @lazyval def gbbq(self): df = GbbqReader().get_df(GBBQ_PATH).query('category == 1') df['datetime'] = pd.to_datetime(df['datetime'], format='%Y%m%d') return df def get_security_type(self, code): if code in self.security_list.code.values: return self.security_list[self.security_list.code == code]['sse'].as_matrix()[0] else: raise SecurityNotExists() def get_security_bars(self, code, freq, index=False): if index: exchange = self.get_security_type(code) func = self.api.get_index_bars else: exchange = get_stock_type(code) func = self.api.get_security_bars df = pd.DataFrame() if freq in ['1d', 'day']: freq = 9 elif freq in ['1m', 'min']: freq = 8 else: raise Exception("1d and 1m frequency supported only") res = [] start = 0 while True: data = func(freq, exchange, code, start, 800) if not data: break res = data + res start += 800 df = self.api.to_df(res).drop( ['year', 'month', 'day', 'hour', 'minute'], axis=1) df['datetime'] = pd.to_datetime(df.datetime) df['code'] = code return df.set_index('datetime') def _get_transaction(self, code, date): res = [] start = 0 while True: data = self.api.get_history_transaction_data( get_stock_type(code), code, start, 2000, date) if not data: break start += 2000 res = data + res if len(res) == 0: return pd.DataFrame() df = self.api.to_df(res).assign(date=date) df.index = pd.to_datetime(str(date) + " " + df["time"]) df['code'] = code return df.drop("time", axis=1) def time_and_price(self, code): start = 0 res = [] exchange = self.get_security_type(code) while True: data = self.api.get_transaction_data(exchange, code, start, 2000) if not data: break res = data + res start += 2000 df = self.api.to_df(res) df.time = pd.to_datetime( str(pd.to_datetime('today').date()) + " " + df['time']) df.loc[0, 'time'] = df.time[1] return df.set_index('time') @classmethod def minute_bars_from_transaction(cls, transaction, freq): if transaction.empty: return pd.DataFrame() data = transaction['price'].resample(freq, label='right', closed='left').ohlc() data['volume'] = transaction['vol'].resample(freq, label='right', closed='left').sum() data['code'] = transaction['code'][0] return fillna(data) def get_k_data(self, code, start, end, freq): if isinstance(start, str) or isinstance(end, str): start = pd.Timestamp(start) end = pd.Timestamp(end) sessions = pd.date_range(start, end) trade_days = map(int, sessions.strftime("%Y%m%d")) if freq == '1m': freq = '1 min' if freq == '1d': freq = '24 H' res = [] for trade_day in trade_days: df = Engine.minute_bars_from_transaction( self._get_transaction(code, trade_day), freq) if df.empty: continue res.append(df) if len(res) != 0: return pd.concat(res) return pd.DataFrame()
class TDXData(DataSource): def __init__(self, db_interface: DBInterface = None, host: str = None, port: int = None): super().__init__(db_interface) if host is None: conf = get_global_config() host = conf['tdx_server']['host'] port = conf['tdx_server']['port'] self.api = TdxHq_API() self.host = host self.port = port self._factor_param = utils.load_param('tdx_param.json') self.stock_ticker = StockTickers(db_interface) def connect(self): self.api.connect(self.host, self.port) def __enter__(self): self.connect() return self def __exit__(self, exc_type, exc_val, exc_tb): self.api.disconnect() def update_stock_minute(self): """更新股票分钟行情""" table_name = '股票分钟行情' db_timestamp = self.db_interface.get_latest_timestamp(table_name, dt.datetime(2015, 1, 1)) start_date = self.calendar.offset(db_timestamp.date(), 1) end_date = dt.datetime.today() dates = self.calendar.select_dates(start_date, end_date) for date in dates: self.get_stock_minute(date) def get_stock_minute(self, date: dt.datetime) -> None: """获取 ``date`` 的股票分钟行情""" minute_data = self._get_stock_minute(date) auction_time = date + dt.timedelta(hours=9, minutes=25) auction_db_data = self.db_interface.read_table('股票集合竞价数据', columns=['成交价', '成交量', '成交额'], dates=auction_time) df = self.left_shift_minute_data(minute_data=minute_data, auction_db_data=auction_db_data) self.db_interface.insert_df(df, '股票分钟行情') def _get_stock_minute(self, date: dt.datetime) -> pd.DataFrame: num_days = self.calendar.days_count(date, dt.date.today()) start_index = num_days * 60 * 4 tickers = self.stock_ticker.ticker(date) storage = [] with tqdm(tickers) as pbar: for ticker in tickers: pbar.set_description(f'下载 {ticker} 在 {date} 的分钟数据') code, market = self._split_ticker(ticker) data = self.api.get_security_bars(category=8, market=market, code=code, start=start_index, count=240) data = self._formatting_data(data, ticker) storage.append(data) pbar.update() df = pd.concat(storage) return df def _formatting_data(self, info: OrderedDict, ticker: str) -> pd.DataFrame: df = pd.DataFrame(info) df['datetime'] = df['datetime'].apply(self.str2datetime) df = df.drop(['year', 'month', 'day', 'hour', 'minute'], axis=1).rename(self._factor_param['行情数据'], axis=1) df['ID'] = ticker df = df.set_index(['DateTime', 'ID'], drop=True) return df @staticmethod def _split_ticker(ticker: str) -> [str, int]: code, market_str = ticker.split('.') market = 0 if market_str == 'SZ' else 1 return code, market @staticmethod def str2datetime(date: str) -> dt.datetime: return dt.datetime.strptime(date, '%Y-%m-%d %H:%M')
class PytdxApi: """TDX数据服务类""" def __init__(self): """Constructor""" self.connect_status: bool = False self.login_status: bool = False self.hq_api = None # 行情API self.conc_code_num = 50 # 并发获取行情的股票个数 # 行情订阅 self.active = False self.run_subscribe = Thread(target=self.get_realtime_data) self.symbols = list() self.symbols_split = list() def connect_api(self, host: str = "", port: int = 0): """连接行情api""" # 连接行情API并检查连接情况 try: if not self.connect_status: self.hq_api = TdxHq_API() self.hq_api.connect(host, port) self.connect_status = True self.login_status = True self.subscribe_start() except Exception as e: return e def get_realtime_quotes(self, quotes_list: list): """获取实时行情数据""" data = self.hq_api.get_security_quotes(quotes_list) return data def get_realtime_data(self): """获取实时行情切片""" try: while self.active: if not self.symbols_split: sleep(1) continue data = list() for symbols in self.symbols_split: d = self.get_realtime_quotes(symbols) data.extend(d) self.on_tick_data(data) sleep(2) except: error = dict() error["error_id"] = "pytdx" error["error_msg"] = "行情订阅失败" self.on_error(error) def get_transaction_count(self, market: int) -> int: """ 查询市场标的数量 """ return self.hq_api.get_security_count(market) def get_transaction_list(self, market: int, start: int) -> list: """查询市场标的列表""" return self.hq_api.get_security_list(market, start) def subscribe_start(self): """启动行情订阅""" self.active = True self.run_subscribe.start() def subscribe(self, symbol: Any): """订阅行情数据""" if isinstance(symbol, tuple): if symbol not in self.symbols: self.symbols.append(symbol) elif isinstance(symbol, list): for s in symbol: if s not in self.symbols: self.symbols.append(s) else: error = dict() error["error_id"] = "pytdx" error["error_msg"] = f"订阅标的代码格式不正确{symbol}" self.on_error(error) return symbol_split = self.get_code_split() self.symbols_split = copy(symbol_split) def subscribe_close(self): """关闭订阅""" if self.active: self.active = False self.run_subscribe.join() def get_transaction_info(self): """获取所有合约信息""" for exchange in list(exchange_map.values()): count = self.get_transaction_count(exchange) for c in range(0, count, 1000): symbols = self.get_transaction_list(exchange, c) for symbol in symbols: symbol["exchange"] = exchange if symbol["code"][:2] in ["60", "30", "688", "00"]: symbol["product"] = 3 else: symbol["product"] = 2 self.on_contract_info(symbol, False) self.on_contract_info({"exchange": exchange}, True) def get_all_stock(self): """获取所有股票数据""" stocks = list() for exchange in list(exchange_map.values()): count = self.get_transaction_count(exchange) for s in range(0, count, 1000): d = self.get_transaction_list(exchange, s) stocks.extend(d) l = len(stocks) - 1 for i, stock in enumerate(stocks): if stock["code"][:1] in ["60", "30", "688", "00"]: if i == l: self.on_contract_info(stock, True) else: self.on_contract_info(stock, False) def on_contract_info(self, data: dict, last: bool) -> None: """""" pass def on_tick_data(self, data): """订阅数据处理""" pass def on_error(self, err): """接口错误处理""" pass @staticmethod def generate_symbols(symbols: list): """组装symbols数据,pytdx接收的是以市场代码和标的代码组成的元祖的list""" new_symbols = [] for symbol in symbols: code, exchange = symbol.split(".") new_symbol = (exchange_map[exchange], code) new_symbols.append(new_symbol) return new_symbols @staticmethod def get_fast_ip(): """获取最快IP""" host = "210.51.39.201" port = 7709 return host, port @staticmethod def check_symbol(symbol: str): """检查标的格式""" if symbol: code, market = symbol.split(".") market = exchange_map.get(market) return code, market else: return False def get_code_split(self): """获得切割好的股票代码段""" code_split_list = [] for i in range(0, len(self.symbols) + 1, self.conc_code_num): code_split = self.symbols[i : i + self.conc_code_num] code_split_list.append(code_split) return code_split_list def exit(self): """数据服务关闭""" # 关闭订阅 self.subscribe_close() # 关闭接口 self.login_status = False self.connect_status = False self.hq_api.disconnect() self.hq_api = None
class Engine: concurrent_thread_count = 50 def __init__(self, *args, **kwargs): if 'ip' in kwargs: self.ip = kwargs.pop('ip') else: if kwargs.pop('best_ip', False): self.ip = self.best_ip else: self.ip = '14.17.75.71' if 'concurrent_thread_count' in kwargs: self.concurrent_thread_count = kwargs.pop( 'concurrent_thread_count', 50) self.thread_num = kwargs.pop('thread_num', 1) self.api = TdxHq_API(args, kwargs, raise_exception=True) def connect(self): self.api.connect(self.ip) return self def __enter__(self): return self def exit(self): self.api.disconnect() def __exit__(self, exc_type, exc_val, exc_tb): self.api.disconnect() def quotes(self, code): code = [code] if not isinstance(code, list) else code code = self.security_list[self.security_list.code.isin( code)].index.tolist() data = [ self.api.to_df( self.api.get_security_quotes(code[80 * pos:80 * (pos + 1)])) for pos in range(int(len(code) / 80) + 1) ] return pd.concat(data) # data = data[['code', 'open', 'high', 'low', 'price']] # data['datetime'] = datetime.datetime.now() # return data.set_index('code', drop=False, inplace=False) def stock_quotes(self): code = self.stock_list.index.tolist() data = [ self.api.to_df( self.api.get_security_quotes(code[80 * pos:80 * (pos + 1)])) for pos in range(int(len(code) / 80) + 1) ] return pd.concat(data) @lazyval def security_list(self): return pd.concat([ pd.concat([ self.api.to_df(self.api.get_security_list( j, i * 1000)).assign(sse=0 if j == 0 else 1).set_index( ['sse', 'code'], drop=False) for i in range(int(self.api.get_security_count(j) / 1000) + 1) ], axis=0) for j in range(2) ], axis=0) @lazyval def stock_list(self): aa = map(stock_filter, self.security_list.index.tolist()) return self.security_list[list(aa)] @lazyval def best_ip(self): return select_best_ip() @lazyval def concept(self): return self.api.to_df( self.api.get_and_parse_block_info(TDXParams.BLOCK_GN)) @lazyval def index(self): return self.api.to_df( self.api.get_and_parse_block_info(TDXParams.BLOCK_SZ)) @lazyval def fengge(self): return self.api.to_df( self.api.get_and_parse_block_info(TDXParams.BLOCK_FG)) @lazyval def block(self): return self.api.to_df( self.api.get_and_parse_block_info(TDXParams.BLOCK_DEFAULT)) @lazyval def customer_block(self): return CustomerBlockReader().get_df(CUSTOMER_BLOCK_PATH) def xdxr(self, code): df = self.api.to_df( self.api.get_xdxr_info(self.get_security_type(code), code)) if df.empty: return df df['datetime'] = pd.to_datetime((df.year * 10000 + df.month * 100 + df.day).apply(lambda x: str(x))) return df.drop(['year', 'month', 'day'], axis=1).set_index('datetime') @lazyval def gbbq(self): df = GbbqReader().get_df(GBBQ_PATH).query('category == 1') df['datetime'] = pd.to_datetime(df['datetime'], format='%Y%m%d') return df def get_security_type(self, code): if code in self.security_list.code.values: return self.security_list[self.security_list.code == code]['sse'].as_matrix()[0] else: raise SecurityNotExists() @retry(3) def get_security_bars(self, code, freq, start=None, end=None, index=False): if index: exchange = self.get_security_type(code) func = self.api.get_index_bars else: exchange = get_stock_type(code) func = self.api.get_security_bars if start: start = start.tz_localize(None) if end: end = end.tz_localize(None) if freq in ['1d', 'day']: freq = 9 elif freq in ['1m', 'min']: freq = 8 else: raise Exception("1d and 1m frequency supported only") res = [] pos = 0 while True: data = func(freq, exchange, code, pos, 800) if not data: break res = data + res pos += 800 if start and pd.to_datetime(data[0]['datetime']) < start: break try: df = self.api.to_df(res).drop( ['year', 'month', 'day', 'hour', 'minute'], axis=1) df['datetime'] = pd.to_datetime(df.datetime) df.set_index('datetime', inplace=True) if freq == 9: df.index = df.index.normalize() except ValueError: # 未上市股票,无数据 logger.warning("no k line data for {}".format(code)) # return pd.DataFrame({ # 'amount': [0], # 'close': [0], # 'open': [0], # 'high': [0], # 'low': [0], # 'vol': [0], # 'code': code # }, # index=[start] # ) return pd.DataFrame() close = [df.close.values[-1]] if start: df = df.loc[lambda df: start <= df.index] if end: df = df.loc[lambda df: df.index.normalize() <= end] if df.empty: # return pd.DataFrame({ # 'amount': [0], # 'close': close, # 'open': close, # 'high': close, # 'low': close, # 'vol': [0], # 'code': code # }, # index=[start] # ) return df else: if int(df['vol'][-1]) <= 0 and end == df.index[-1] and len( df) == 1: # 成交量为0,当天返回的是没开盘的数据 return pd.DataFrame() df['code'] = code return df def _get_transaction(self, code, date): res = [] start = 0 while True: data = self.api.get_history_transaction_data( get_stock_type(code), code, start, 2000, date) if not data: break start += 2000 res = data + res if len(res) == 0: return pd.DataFrame() df = self.api.to_df(res).assign(date=date) df.loc[0, 'time'] = df.time[1] df.index = pd.to_datetime(str(date) + " " + df["time"]) df['code'] = code return df.drop("time", axis=1) def time_and_price(self, code): start = 0 res = [] exchange = self.get_security_type(code) while True: data = self.api.get_transaction_data(exchange, code, start, 2000) if not data: break res = data + res start += 2000 df = self.api.to_df(res) df.time = pd.to_datetime( str(pd.to_datetime('today').date()) + " " + df['time']) df.loc[0, 'time'] = df.time[1] return df.set_index('time') @classmethod def minute_bars_from_transaction(cls, transaction, freq): if transaction.empty: return pd.DataFrame() mask = transaction.index < transaction.index[0].normalize( ) + pd.Timedelta('12 H') def resample(transaction): if transaction.empty: return pd.DataFrame() data = transaction['price'].resample(freq, label='right', closed='left').ohlc() data['volume'] = transaction['vol'].resample(freq, label='right', closed='left').sum() data['code'] = transaction['code'][0] return data morning = resample(transaction[mask]) afternoon = resample(transaction[~mask]) if morning.empty and afternoon.empty: return pd.DataFrame() if not afternoon.empty: morning.index.values[-1] = afternoon.index[0] - pd.Timedelta( '1 min') df = pd.concat([morning, afternoon]) return fillna(df) def _get_k_data(self, code, freq, sessions): trade_days = map(int, sessions.strftime("%Y%m%d")) if freq == '1m': freq = '1 min' if freq == '1d': freq = '24 H' res = [] concurrent_count = self.concurrent_thread_count jobs = [] for trade_day in trade_days: # df = Engine.minute_bars_from_transaction(self._get_transaction(code, trade_day), freq) reqevent = gevent.spawn(Engine.minute_bars_from_transaction, self._get_transaction(code, trade_day), freq) jobs.append(reqevent) if len(jobs) >= concurrent_count: gevent.joinall(jobs, timeout=30) for j in jobs: if j.value is not None and not j.value.empty: res.append(j.value) jobs.clear() gevent.joinall(jobs, timeout=30) for j in jobs: if j.value is not None and not j.value.empty: res.append(j.value) jobs.clear() if len(res) != 0: return pd.concat(res) return pd.DataFrame() def get_k_data(self, code, start, end, freq, check=True): if isinstance(start, str) or isinstance(end, str): start = pd.Timestamp(start) end = pd.Timestamp(end) if check: daily_bars = self.get_security_bars(code, '1d', start, end) if daily_bars is None or daily_bars.empty: return daily_bars sessions = daily_bars.index else: sessions = pd.bdate_range(start, end, weekmask='Mon Tue Wed Thu Fri') df = self._get_k_data(code, freq, sessions) def check_df(freq, df, daily_bars): if freq == '1m': need_check = pd.DataFrame({ 'open': df['open'].resample('1D').first(), 'high': df['high'].resample('1D').max(), 'low': df['low'].resample('1D').min(), 'close': df['close'].resample('1D').last(), 'volume': df['volume'].resample('1D').sum() }).dropna() else: need_check = df if daily_bars.shape[0] != need_check.shape[0]: logger.warning("{} merged {}, expected {}".format( code, need_check.shape[0], daily_bars.shape[0])) need_check = fillna( need_check.reindex(daily_bars.index, copy=False)) diff = daily_bars[['open', 'close']] == need_check[['open', 'close']] res = (diff.open) & (diff.close) sessions = res[res == False].index return sessions if not df.empty: if check: sessions = check_df(freq, df, daily_bars) if sessions.shape[0] != 0: logger.info( "fixing data for {}-{} with sessions: {}".format( code, freq, sessions)) fix = self._get_k_data(code, freq, sessions) df.loc[fix.index] = fix return df return df
class SP(object): def __init__(self,userid="account4",rate=1.2,products="1",limit=2,total=1000000,mock=True,server="http://192.168.118.1:65000"): if products == "1": self.products = {'880414': {'args': (3, 12, 90), 'stocklst': {}}, '880456': {'args': (6, 28, 40), 'stocklst': {}}, '880476': {'args': (5, 20, 90), 'stocklst': {}}, '880440': {'args': (4, 8, 85), 'stocklst': {}}, '880424': {'args': (4, 16, 110),'stocklst': {}}, '880448': {'args': (3, 14, 60), 'stocklst': {}}, '880454': {'args': (4, 8, 85), 'stocklst': {}}, '880493': {'args': (3, 24, 55), 'stocklst': {}}, '880344': {'args': (4, 4, 30), 'stocklst': {}}, '880301': {'args': (6, 20, 30), 'stocklst': {}}, '880464': {'args': (4, 14, 50), 'stocklst': {}}, '880459': {'args': (3, 16, 80), 'stocklst': {}}, '880380': {'args': (5, 22, 70), 'stocklst': {}}, '880472': {'args': (3, 32, 170),'stocklst': {}}, '880421': {'args': (3, 16, 45), 'stocklst': {}}, '880471': {'args': (4, 12, 40), 'stocklst': {}}, '880453': {'args': (6, 14, 30), 'stocklst': {}}, '880350': {'args': (4, 26, 30), 'stocklst': {}}, '880447': {'args': (5, 20, 170),'stocklst': {}}, '880351': {'args': (3, 6, 65), 'stocklst': {}}, '880390': {'args': (3, 14, 65), 'stocklst': {}}, '880406': {'args': (4, 16, 50), 'stocklst': {}}, '880305': {'args': (3, 14, 95), 'stocklst': {}}, '880492': {'args': (3, 30, 105),'stocklst': {}}, '880387': {'args': (5, 20, 70), 'stocklst': {}}, '880418': {'args': (4, 16, 100),'stocklst': {}}, '880367': {'args': (5, 14, 110),'stocklst': {}}, '880398': {'args': (5, 26, 100),'stocklst': {}}, '880437': {'args': (4, 8, 85), 'stocklst': {}}, '880474': {'args': (4, 24, 45), 'stocklst': {}}, '880324': {'args': (6, 22, 75), 'stocklst': {}}, '880335': {'args': (3, 12, 75), 'stocklst': {}}, '880372': {'args': (6, 20, 105),'stocklst': {}}, '880491': {'args': (4, 16, 20), 'stocklst': {}}, '880490': {'args': (3, 26, 110),'stocklst': {}}, '880431': {'args': (7, 12, 40), 'stocklst': {}}, '880432': {'args': (3, 10, 110),'stocklst': {}}, '880318': {'args': (4, 20, 90), 'stocklst': {}}, '880497': {'args': (3, 8, 40), 'stocklst': {}}, '880494': {'args': (3, 30, 110),'stocklst': {}}, '880400': {'args': (3, 10, 90), 'stocklst': {}}, '880360': {'args': (3, 16, 70), 'stocklst': {}}, '880310': {'args': (3, 12, 20), 'stocklst': {}}, '880423': {'args': (5, 16, 30), 'stocklst': {}}, '880430': {'args': (7, 12, 25), 'stocklst': {}}, '880422': {'args': (6, 8, 35), 'stocklst': {}}, '880465': {'args': (4, 12, 95), 'stocklst': {}}, '880482': {'args': (3, 20, 180),'stocklst': {}}, '880355': {'args': (3, 18, 45), 'stocklst': {}}, '880473': {'args': (3, 6, 40), 'stocklst': {}}, '880446': {'args': (3, 8, 45), 'stocklst': {}}, '880452': {'args': (3, 32, 90), 'stocklst': {}}, '880455': {'args': (5, 6, 20), 'stocklst': {}}, '880399': {'args': (3, 8, 85), 'stocklst': {}}, '880330': {'args': (5, 12, 80), 'stocklst': {}}, '880489': {'args': (3, 18, 190),'stocklst': {}}} elif products == "2": #资金比较少是,选择配置部分行业 self.products = {'880414': {'args': (3, 12, 90), 'stocklst': {}}, '880456': {'args': (6, 28, 40), 'stocklst': {}}, '880476': {'args': (5, 20, 90), 'stocklst': {}}, '880440': {'args': (4, 8, 85), 'stocklst': {}}, '880424': {'args': (4, 16, 110),'stocklst': {}}, '880448': {'args': (3, 14, 60), 'stocklst': {}}, '880454': {'args': (4, 8, 85), 'stocklst': {}}, '880493': {'args': (3, 24, 55), 'stocklst': {}}, '880344': {'args': (4, 4, 30), 'stocklst': {}}, '880301': {'args': (6, 20, 30), 'stocklst': {}}, '880464': {'args': (4, 14, 50), 'stocklst': {}}, '880459': {'args': (3, 16, 80), 'stocklst': {}}, '880380': {'args': (5, 22, 70), 'stocklst': {}}, '880472': {'args': (3, 32, 170),'stocklst': {}}, '880421': {'args': (3, 16, 45), 'stocklst': {}}, '880471': {'args': (4, 12, 40), 'stocklst': {}}, '880453': {'args': (6, 14, 30), 'stocklst': {}}, '880350': {'args': (4, 26, 30), 'stocklst': {}}, '880447': {'args': (5, 20, 170),'stocklst': {}}, '880351': {'args': (3, 6, 65), 'stocklst': {}}, '880390': {'args': (3, 14, 65), 'stocklst': {}}, '880406': {'args': (4, 16, 50), 'stocklst': {}}, '880305': {'args': (3, 14, 95), 'stocklst': {}}, '880492': {'args': (3, 30, 105),'stocklst': {}}, '880387': {'args': (5, 20, 70), 'stocklst': {}}, '880418': {'args': (4, 16, 100),'stocklst': {}}, } self.server = server self.datatype = 1 self.userid = userid self.limit = limit self.api = TdxHq_API(heartbeat=True) self.trader = None self.trading = False self.mock = mock self.rate = rate #持仓杠杆率 self.total = total #默认持仓资金 self.TDX_IP_SETS = STOCK_IP_SETS self.file_incon = FILE_INCON self.file_tdxhy = FILE_TDXHY self.file_tdxzs = FILE_TDXZS def connect(self): for ip in self.TDX_IP_SETS: try: if self.api.connect(ip, 7709): return except: pass def getdata(self,product,market=1,number=5000,pn=400): data = [] for i in range(int(number/pn)+1): temp = self.api.get_index_bars(self.datatype, market, product, (int(number/pn)-i)*pn,pn) if not temp or len(temp)<pn: self.connect() for _ in range(2): temp = self.api.get_index_bars(self.datatype, market, product, (int(number/pn)-i)*pn,pn) if not temp or len(temp)<pn: logger.info("record not reach the limit!") else: break data += temp df = self.api.to_df(data)[["open","close","high","low","datetime"]] df.set_index("datetime",inplace=True,drop=False) return df def disconnect(self): logger.info("[DISCONNECT]:statrt disconnect!!!!!") if self.istradeday: self.api.disconnect() logger.info("[DISCONNECT]:disconnect finished!!!!!") def updatetotal(self): '''更新总资金 ''' accountinfo,holdlists = self.trader.position() nhg = holdlists[holdlists[u"证券代码"].map(lambda x:x in ["SZRQ88","SHRQ88"])][u"最新市值"].sum() self.total = accountinfo.ix["总资产"]["人民币"]-nhg return self.total def set_permoney(self): '''单个品种资金上限 ''' self.permoney = self.total * self.rate /len(self.products) return self.permoney def _get_incon(self,): '''获取行业分类代码 ''' f= open(self.file_incon, "rb") data = f.read() strings = data.decode("gbk", 'ignore').rstrip("\x00").replace("\r\n","\n") data = strings.split("######") rst = {} for hystr in data: key = re.findall(r'#.*',hystr) if key == ['#TDXNHY']: hylst = hystr.replace("#TDXNHY","").strip("\n").split("\n") for item in hylst: k,v = item.split("|") rst[k] = [v] return rst def _get_tdxhy(self,islocal=True): '''获取股票和行业对应列表 ''' if islocal: stocklist = HY_WEIGHT.keys() else: stocklist = list(ts.get_stock_basics().index) #获取全市场股票代码 rst = self._get_incon() f= open(self.file_tdxhy, "rb") data = f.read().decode("gbk", 'ignore').rstrip("\x00").replace("\r\n","\n").strip("\n").split("\n") for i in data: _,code,tdxhy,_,_ = i.split("|") if tdxhy != "T00" and code in stocklist: rst[tdxhy].append(code) return rst def _get_tdxzs(self,islocal=True): '''生成通达性版块代码对应股票列表 ''' dct = {} rst = self._get_tdxhy(islocal=islocal) f= open(self.file_tdxzs, "rb") data = f.read().decode("gbk", 'ignore').rstrip("\x00").replace("\r\n","\n").strip("\n").split("\n") for i in data: name,code,_,_,_,hy = i.split("|") code = int(code) if 880301<=code and 880497>=code and hy in rst.keys() : k = hy[:5] if not dct.__contains__(k): dct[k] = {"name":"","code":"","stocklist":[]} if k==hy: dct[k]["name"] = name dct[k]["code"] = code dct[k]["stocklist"].extend(rst[hy][1:]) return dct def get_tdxhy_list(self,islocal=True): '''获取通达信行业板块指数对应的股票列表 ''' return self._get_tdxzs(islocal) def get_weight(self,htlist={},islocal=True): '''获取行业板块个股权重,流动市值为权重系数 备注:回测是为方便处理,以最后一天的权重系数作为历史上的权重 ''' if islocal: self.weight = HY_WEIGHT else: if not htlist: htlist = self.get_tdxhy_list(islocal) tasks = [] for v in htlist.values(): tasks.append(gevent.spawn(self.get_latest_ltsz,v["stocklist"])) gevent.joinall(tasks) return self.weight def get_latest_ltsz(self,stocks=[]): '''获取最新流通市值,千万为单位,取整 ''' unit = 10000000 for code in stocks: mk = self._select_market_code(code) print(mk,code) try: ltgb = self.api.get_finance_info(mk,code)["liutongguben"] price = self.api.get_security_bars(4,mk,code,0,1)[0]["close"] ltsz = int(ltgb*price/unit) self.weight[code] = ltsz except: print("*****",code) return def set_instrument(self): '''设置交易股票 ''' func = lambda x :1 if x.startswith("6") else 0 weight = self.get_weight() for v in self.get_tdxhy_list().values(): code = str(v["code"]) if not self.products.__contains__(code):continue stocks = [(i,weight[i]) for i in v["stocklist"]] limit_stocks = sorted(stocks,key=lambda x:x[1],reverse=True)[:self.limit] total = sum([i[1] for i in limit_stocks]) for i in limit_stocks: market = func(i[0]) price = self.api.get_security_bars(4,market,i[0],0,1)[0]["close"] number = int(self.permoney*i[1]/total/price/100)*100 self.products[code]["stocklst"][i[0]] = number return self.products def judgetradeday(self,): today = datetime.datetime.today().date().strftime('%Y-%m-%d') df = ts.trade_cal() return df[(df["calendarDate"]==today)].isOpen.values[0] @property def istradeday(self): if not hasattr(self, "_istradeday"): self._istradeday = self.judgetradeday() return self._istradeday def initial(self): '''每天初始化设置 ''' logger.info("[INITIAL]:start initial !!!!!!") if not self.istradeday: self.trading = False return self.trading = True logger.info("[INITIAL]:try to create connect... ") self.connect() self.trader = trade(UserID=self.userid,api=self.api,mock=self.mock,server=self.server) logger.info("[INITIAL]:connect successful!") logger.info("[INITIAL]:initial account info...") self.updatetotal() #更新账户总资金 self.set_permoney() #设置单个品种资金上限 logger.info("[INITIAL]:set per product money limit:{}".format(self.permoney)) self.set_instrument() #设置交易股票和手数 logger.info("[INITIAL]:set stock list succcessful !!!") logger.info("[INITIAL]:initial finished!!!!!!") def handledata(self,df,args=[]): df.loc[:,"number"] = range(df.shape[0]) s,m,l = args for i in args:#5 15 60 D key = str(5*i) df.loc[:,key+"high"] = df["high"].rolling(i).max() df.loc[:,key+"low"] = df["low"].rolling(i).min() df.loc[:,key+"atr"] = (df[key+"high"]-df[key+"low"]).rolling(10*i).mean() df.loc[:,key+"med"] = (df[key+"high"]+df[key+"low"])/2 df.loc[:,key+"HH"] = df[key+"med"] + 1.5*df[key+"atr"] df.loc[:,key+"LL"] = df[key+"med"] - 1.5*df[key+"atr"] df.loc[:,key+'HHmax'] = df[key+'HH'].rolling(10*i).max() df.loc[:,key+'LLmin'] = df[key+'LL'].rolling(10*i).min() df.loc[df[key+'HH']>=df[key+'HHmax'],key+'hmark'] = df["number"] df.loc[df[key+'LL']<=df[key+'LLmin'],key+'lmark'] = df["number"] df[key+'hmark'].fillna(method="ffill",inplace=True) df[key+'hmark'].fillna(0,inplace=True) df[key+'lmark'].fillna(method="ffill",inplace=True) df[key+'lmark'].fillna(0,inplace=True) df.loc[:,key+'UP'] = df[key+'hmark'] >= df[key+'lmark'] # debuginfo.append({key+'hmark':df.iloc[-1][key+'hmark'],key+'lmark':df.iloc[-1][key+'lmark']}) df.fillna(method="ffill",inplace=True) df.dropna(inplace=True) # logger.info("trademessage:{}".format(debuginfo)) result = (df.iloc[-1][str(5*l)+"UP"] >0)&(df.iloc[-1][str(5*s)+"UP"]>0) result |= (df.iloc[-1][str(5*l)+"UP"]>0)&(df.iloc[-1][str(5*m)+"UP"]>0) result |= (df.iloc[-1][str(5*l)+"UP"]<=0)&(df.iloc[-1][str(5*m)+"UP"]>0)&(df.iloc[-1][str(5*s)+"UP"]>0) return result def sync(self,idx,director=True): stocks = self.products[idx]["stocklst"] for stock,number in stocks.items(): if not director: number = 0 #空信号,清仓 #判断现有持仓 try: code = stock h_number = self.hd_df.ix[code]["证券数量"] except: h_number = 0 logger.info("[RUN]:{},{},{}".format(stock,h_number,number)) #补仓差 cangcha = int((number-h_number)/100)*100 if h_number>0 and abs(cangcha)/h_number<0.2: #如果有持仓,同时仓差小于10% 不进行更改,为了处理频繁加减仓达到问题 continue if cangcha>0: logger.info("[RUN]:buy code:{}, number:{}".format(stock,number-h_number)) self.buy(stock,cangcha) elif cangcha<0: couldsell = self.hd_df.ix[code]["可卖数量"] logger.info("[RUN]:sell code:{}, number:{},couldsell:{}".format(stock,h_number-number,couldsell)) if couldsell >0: self.sell(stock,min(-cangcha,couldsell)) # def buy(self,stock,number): self.trader.buy(stock, number) def sell(self,stock,number): self.trader.sell(stock, number) def check_position(self,status): '''检查仓位情况 ''' self.handleposition() handlelist = [] for ins,v in status.items(): if ins not in self.g_df.index : handlelist.append(ins) elif self.g_df.ix[ins]["Position"] != v["number"]: handlelist.append(ins) return handlelist def handleposition(self): '''计算多仓,空仓,以及昨仓和今仓 ''' self.trader.cancelorder() #先尝试撤单 _,holdlists = self.trader.position() self.hd_df = holdlists if holdlists.shape[0]>0: self.hd_df.set_index("证券代码",inplace=True) self.hd_df.index = self.hd_df.index.astype(np.str).map(lambda x:x if len(x)>=6 else "0"*(6-len(x))+x) return self.hd_df def run(self): logger.info("[RUN]:start run !!!!!") if not self.trading: return self.handleposition() rst = {} for idx in list(self.products.keys()): director = self.handledata(self.getdata(idx,market=1),self.products[idx]["args"]) #用指数出信号 logger.info("[RUN]:trademessage: block:{}, director:{}".format(idx,director)) self.sync(idx,director) rst[idx] = {"up":director,"number":self.products[idx]["stocklst"],"product":idx} logger.info("[RUN]:lastest position status:{}".format(rst)) logger.info("[RUN]:run finished !!!!!")
def __call__(self): total_count = 0 try: connect = sqlite3.connect(self.sqlitefile) except Exception as e: #self.queue.put([self.msg_name, str(e), -1, 0, total_count]) self.queue.put([self.msg_name, 'INFO', str(e), 0, 0]) self.queue.put([self.msg_name, '', 0, None, total_count]) return try: download_dir = self.dest_dir + "/downloads" if not os.path.lexists(download_dir): os.makedirs(download_dir) self.queue.put([self.msg_name, '正在下载钱龙权息信息...', 0, 0, 0]) net_file = urllib.request.urlopen( 'http://www.qianlong.com.cn/download/history/weight.rar', timeout=60) buffer = net_file.read() self.queue.put( [self.msg_name, '钱龙权息信息下载完成,正在校验是否存在更新...', 0, 0, 0]) new_md5 = hashlib.md5(buffer).hexdigest() dest_filename = download_dir + '/weight.rar' old_md5 = None if os.path.exists(dest_filename): with open(dest_filename, 'rb') as oldfile: old_md5 = hashlib.md5(oldfile.read()).hexdigest() #如果没变化不需要解压导入 if new_md5 != old_md5: with open(dest_filename, 'wb') as file: file.write(buffer) self.queue.put([self.msg_name, '正在解压钱龙权息信息...', 0, 0, 0]) x = os.system('unrar x -o+ -inul {} {}'.format( dest_filename, download_dir)) if x != 0: raise Exception("无法找到unrar命令!") self.queue.put([self.msg_name, '正在导入钱龙权息数据...', 0, 0, 0]) total_count = qianlong_import_weight(connect, download_dir + '/weight', 'SH') total_count += qianlong_import_weight(connect, download_dir + '/weight', 'SZ') self.queue.put( [self.msg_name, '导入钱龙权息数据完毕!', 0, 0, total_count]) else: self.queue.put([self.msg_name, '钱龙权息数据无变化', 0, 0, 0]) hosts = search_best_tdx() api = TdxHq_API() api.connect(hosts[0][2], hosts[0][3]) self.queue.put([self.msg_name, '下载通达信权息信息(上证)...', 0, 0, 0]) x = pytdx_import_finance(connect, api, "SH") self.queue.put([self.msg_name, '下载通达信权息信息(深证)...', 0, 0, 0]) x += pytdx_import_finance(connect, api, "SZ") self.queue.put([self.msg_name, '导入通达信权息信息完毕!', 0, 0, x]) api.disconnect() except Exception as e: #self.queue.put([self.msg_name, str(e), -1, 0, total_count]) self.queue.put([self.msg_name, 'INFO', str(e), 0, 0]) finally: connect.commit() connect.close() self.queue.put([self.msg_name, '', 0, None, total_count])
net_file = urllib.request.urlopen('http://www.qianlong.com.cn/download/history/weight.rar', timeout=60) dest_filename = dest_dir + '/weight.rar' with open(dest_filename, 'wb') as file: file.write(net_file.read()) print("下载完成,正在解压...") os.system('unrar x -o+ -inul {} {}'.format(dest_filename, dest_dir)) print("解压完成,正在导入...") add_count = qianlong_import_weight(connect, dest_dir + '/weight', 'SH') add_count += qianlong_import_weight(connect, dest_dir + '/weight', 'SZ') print("导入数量:", add_count) print("\n导入上证分笔数据") #add_count = import_trans(connect, 'SH', quotations, api, dest_dir, max_days=2, progress=ProgressBar) print("\n导入数量:", add_count) print("\n导入上证分时数据") add_count = import_time(connect, 'SZ', quotations, api, dest_dir, max_days=3, progress=ProgressBar) print("\n导入数量:", add_count) """ api.disconnect() connect.close() endtime = time.time() print("\nTotal time:") print("%.2fs" % (endtime - starttime)) print("%.2fm" % ((endtime - starttime) / 60))
def __call__(self): total_count = 0 try: connect = sqlite3.connect(self.sqlitefile) except Exception as e: #self.queue.put([self.msg_name, str(e), -1, 0, total_count]) self.queue.put([self.msg_name, 'INFO', str(e), 0, 0]) self.queue.put([self.msg_name, '', 0, None, total_count]) return try: download_dir = self.dest_dir + "/downloads" if not os.path.lexists(download_dir): os.makedirs(download_dir) self.queue.put([self.msg_name, '正在下载钱龙权息信息...', 0, 0, 0]) net_file = urllib.request.urlopen('http://www.qianlong.com.cn/download/history/weight.rar', timeout=60) buffer = net_file.read() self.queue.put([self.msg_name, '钱龙权息信息下载完成,正在校验是否存在更新...', 0, 0, 0]) new_md5 = hashlib.md5(buffer).hexdigest() dest_filename = download_dir + '/weight.rar' old_md5 = None if os.path.exists(dest_filename): with open(dest_filename, 'rb') as oldfile: old_md5 = hashlib.md5(oldfile.read()).hexdigest() #如果没变化不需要解压导入 if new_md5 != old_md5: with open(dest_filename, 'wb') as file: file.write(buffer) self.queue.put([self.msg_name, '正在解压钱龙权息信息...', 0, 0, 0]) x = os.system('unrar x -o+ -inul {} {}'.format(dest_filename, download_dir)) if x != 0: raise Exception("无法找到unrar命令!") self.queue.put([self.msg_name, '正在导入钱龙权息数据...', 0, 0, 0]) total_count = qianlong_import_weight(connect, download_dir + '/weight', 'SH') total_count += qianlong_import_weight(connect, download_dir + '/weight', 'SZ') self.queue.put([self.msg_name, '导入钱龙权息数据完毕!', 0, 0, total_count]) else: self.queue.put([self.msg_name, '钱龙权息数据无变化', 0, 0, 0]) hosts = search_best_tdx() api = TdxHq_API() api.connect(hosts[0][2], hosts[0][3]) self.queue.put([self.msg_name, '下载通达信权息信息(上证)...', 0, 0, 0]) x = pytdx_import_finance(connect, api, "SH") self.queue.put([self.msg_name, '下载通达信权息信息(深证)...', 0, 0, 0]) x += pytdx_import_finance(connect, api, "SZ") self.queue.put([self.msg_name, '导入通达信权息信息完毕!', 0, 0, x]) api.disconnect() except Exception as e: #self.queue.put([self.msg_name, str(e), -1, 0, total_count]) self.queue.put([self.msg_name, 'INFO', str(e), 0, 0]) finally: connect.commit() connect.close() self.queue.put([self.msg_name, '', 0, None, total_count])
def __call__(self): total_count = 0 try: if self.config.getboolean('hdf5', 'enable', fallback=True): sqlite_file = "{}/stock.db".format(self.config['hdf5']['dir']) connect = sqlite3.connect(sqlite_file, timeout=1800) pytdx_import_weight = pytdx_import_weight_to_sqlite self.logger.debug('use sqlite import weight') else: db_config = { 'user': self.config['mysql']['usr'], 'password': self.config['mysql']['pwd'], 'host': self.config['mysql']['host'], 'port': self.config['mysql']['port'] } connect = mysql.connector.connect(**db_config) pytdx_import_weight = pytdx_import_weight_to_mysql self.logger.debug('use mysql import weight') except Exception as e: #self.queue.put([self.msg_name, str(e), -1, 0, total_count]) self.queue.put([self.msg_name, 'INFO', str(e), 0, 0]) self.queue.put([self.msg_name, '', 0, None, total_count]) return try: """ download_dir = self.dest_dir + "/downloads" if not os.path.lexists(download_dir): os.makedirs(download_dir) self.queue.put([self.msg_name, '正在下载钱龙权息信息...', 0, 0, 0]) net_file = urllib.request.urlopen( 'http://www.qianlong.com.cn/download/history/weight.rar', timeout=60 ) buffer = net_file.read() self.queue.put([self.msg_name, '钱龙权息信息下载完成,正在校验是否存在更新...', 0, 0, 0]) new_md5 = hashlib.md5(buffer).hexdigest() dest_filename = download_dir + '/weight.rar' old_md5 = None if os.path.exists(dest_filename): with open(dest_filename, 'rb') as oldfile: old_md5 = hashlib.md5(oldfile.read()).hexdigest() #如果没变化不需要解压导入 if new_md5 != old_md5: with open(dest_filename, 'wb') as file: file.write(buffer) self.queue.put([self.msg_name, '正在解压钱龙权息信息...', 0, 0, 0]) x = os.system('unrar x -o+ -inul {} {}'.format(dest_filename, download_dir)) if x != 0: raise Exception("无法找到unrar命令!") self.queue.put([self.msg_name, '正在导入钱龙权息数据...', 0, 0, 0]) total_count = qianlong_import_weight(connect, download_dir + '/weight', 'SH') total_count += qianlong_import_weight(connect, download_dir + '/weight', 'SZ') self.queue.put([self.msg_name, '导入钱龙权息数据完毕!', 0, 0, total_count]) else: self.queue.put([self.msg_name, '钱龙权息数据无变化', 0, 0, 0]) """ hosts = search_best_tdx() api = TdxHq_API() api.connect(hosts[0][2], hosts[0][3]) self.queue.put([self.msg_name, '正在导入权息数据...', 0, 0, 0]) total_count = pytdx_import_weight(api, connect, "SH") total_count += pytdx_import_weight(api, connect, "SZ") self.queue.put([self.msg_name, '导入权息数据完毕!', 0, 0, total_count]) #self.queue.put([self.msg_name, '下载通达信财务信息(上证)...', 0, 0, 0]) #x = pytdx_import_finance(connect, api, "SH") #self.queue.put([self.msg_name, '下载通达信财务信息(深证)...', 0, 0, 0]) #x += pytdx_import_finance(connect, api, "SZ") #self.queue.put([self.msg_name, '导入通达信财务信息完毕!', 0, 0, x]) api.disconnect() except Exception as e: self.logger.error(e) #self.queue.put([self.msg_name, str(e), -1, 0, total_count]) self.queue.put([self.msg_name, 'INFO', str(e), 0, 0]) finally: connect.commit() connect.close() self.queue.put([self.msg_name, '', 0, None, total_count])
'changqifuzhai': '长期负债', 'zibengongjijin': '资本公积金', 'jingzichan': '净资产', 'zhuyingshouru': '主营收入', 'zhuyinglirun': '主营利润', 'yingshouzhangkuan': '应收账款', 'yingyelirun': '营业利润', 'touzishouyu': '投资收益', 'jingyingxianjinliu': '经营现金流', 'zongxianjinliu': '总现金流', 'cunhuo': '存货', 'lirunzonghe': '利润总和', 'shuihoulirun': '税后利润', 'jinglirun': '净利润', 'weifenpeilirun': '未分配利润', 'meigujingzichan': '每股净资产', 'baoliu2': 'baoliu2' }) #print finance_df #finance_df.to_csv(r"..\all_other_data\test_finance_info.csv", encoding="gbk", quoting=csv.QUOTE_NONE, index=False) # string_columns = ['代码'] # finance_df[string_columns] = finance_df[string_columns].applymap( # lambda x: '=""' if type(x) is float else '="' + str(x) + '"') finance_df.to_csv(r"..\all_other_data\all_finance_info.csv", encoding="gbk", quoting=csv.QUOTE_NONE, index=False) TDXHQ.disconnect()
class TdxHelper: ip_list = [{ 'ip': '119.147.212.81', 'port': 7709 }, { 'ip': '60.12.136.250', 'port': 7709 }] def __init__(self): #连接tdx接口 self.api = TdxHq_API() if not self.api.connect('60.12.136.250', 7709): print("服务器连接失败!") # pandas数据显示设置 pd.set_option('display.max_columns', None) # 显示所有列 #pd.set_option('display.max_rows', None) # 显示所有行 # mysql对象 self.mysql = mysqlHelper(config.mysql_host, config.mysql_username, bluedothe.mysql_password, config.mysql_dbname) # pandas的mysql对象 self.engine = create_engine( f'mysql+pymysql://{config.mysql_username}:{bluedothe.mysql_password}@{config.mysql_host}/{config.mysql_dbname}?charset=utf8' ) #断开tdx接口连接 def close_connect(self): self.api.disconnect() #获取k线,最后一个参数day,说明需要获取的数量,本接口只获取从最近交易日往前的数据 #输入参数:五个参数分别为:category(k线),市场代码(0:深圳,1:上海),股票代码,开始位置(从最近交易日向前取,0表示最近交易日),返回的记录条数 #K线种类: 0 5分钟K线; 1 15分钟K线; 2 30分钟K线; 3 1小时K线; 4 日K线;5 周K线;6 月K线;7 1分钟;8 1分钟K线; 9 日K线;10 季K线;11 年K线 #返回值:open,close,high,low,vol,amount,year,month,day,hour,minute,datetime # csv格式:code,ts_code,trade_date(缩写),trade_time,time_index,open,high,low,close,amount,volume def get_security_bars(self, category, market, code, start=0, count=240): dict = {0: 'SZ', 1: 'SH'} ts_code = code + "." + dict[market] order = [ 'code', 'ts_code', 'trade_date', 'trade_time', 'time_index', 'open', 'high', 'low', 'close', 'amount', 'volume' ] #df = self.api.get_security_bars(9, 0, '000001', 0, 10) # 返回普通list df = self.api.to_df( self.api.get_security_bars(category, market, code, start, count)) # 返回DataFrame if df.empty: return df df.insert(0, 'ts_code', ts_code) df.insert(0, 'code', code) df['trade_time'] = df['datetime'].apply(lambda x: str(x)[11:19]) df['time_index'] = df['trade_time'].apply( lambda x: datatime_util.stockTradeTime2Index(x)) df['trade_date'] = df['datetime'].apply( lambda x: (str(x)[0:10]).replace('-', '')) df.rename(columns={'vol': 'volume'}, inplace=True) df.drop(['year', 'month', 'day', 'hour', 'minute', 'datetime'], axis=1, inplace=True) df['volume'] = df['volume'].apply(lambda x: int(x)) #取整 df.loc[df['amount'] == 5.877471754111438e-39, 'amount'] = 0 #列值根据条件筛选后修改为0 df = df[order] filename = config.tdx_csv_minline1_all + ts_code + ".csv" if os.path.isfile(filename): df.to_csv(filename, index=False, mode='a', header=False, sep=',', encoding="utf_8_sig") else: df.to_csv(filename, index=False, mode='w', header=True, sep=',', encoding="utf_8_sig") print("新增加的一分钟all股票数据:", filename) # 获取1分钟k线,最后一个参数说明需要获取的数量,本接口只获取从最近交易日往前的数据 # 输入参数:五个参数分别为:category(k线),市场代码(0:深圳,1:上海),股票代码,开始位置(从最近交易日向前取,0表示最近交易日),返回的记录条数 # K线种类: 0 5分钟K线; 1 15分钟K线; 2 30分钟K线; 3 1小时K线; 4 日K线;5 周K线;6 月K线;7 1分钟;8 1分钟K线; 9 日K线;10 季K线;11 年K线 # 返回值:open,close,high,low,vol,amount,year,month,day,hour,minute,datetime # csv格式:code,ts_code,trade_date(缩写),trade_time,time_index,open,high,low,close,amount,volume def get_security_bars_minute1(self, category, market, code, start, count): dict = {0: 'SZ', 1: 'SH'} ts_code = code + "." + dict[market] order = [ 'code', 'ts_code', 'trade_date', 'trade_time', 'time_index', 'open', 'high', 'low', 'close', 'amount', 'volume' ] # df = self.api.get_security_bars(9, 0, '000001', 0, 10) # 返回普通list df = self.api.to_df( self.api.get_security_bars(category, market, code, start, count)) # 返回DataFrame if df.empty: return df.insert(0, 'ts_code', ts_code) df.insert(0, 'code', code) df['trade_time'] = df['datetime'].apply(lambda x: str(x)[11:19]) df['time_index'] = df['trade_time'].apply( lambda x: datatime_util.stockTradeTime2Index(x)) df['trade_date'] = df['datetime'].apply( lambda x: (str(x)[0:10]).replace('-', '')) df.rename(columns={'vol': 'volume'}, inplace=True) df.drop(['year', 'month', 'day', 'hour', 'minute', 'datetime'], axis=1, inplace=True) df['volume'] = df['volume'].apply(lambda x: int(x)) # 取整 df.loc[df['amount'] == 5.877471754111438e-39, 'amount'] = 0 # 列值根据条件筛选后修改为0 df = df[order] #过滤掉停牌的数据,在tdx中,停牌股票也能取到数据,价格是前一交易日的收盘价,所以只能用成交量或成交金额为0来判断 #1按日期分组后取出成交量为0的日期;2循环过滤掉成交量为0的日期的数据。 dfg = df.groupby(by='trade_date').mean() #分组 dfg['trade_date'] = dfg.index dfg = dfg[dfg.volume == 0] #条件过滤,保留满足条件的数据 for trade_date in dfg['trade_date'].values: df = df[(df['trade_date'] != trade_date)] # 每个条件要用括号()括起来 return df #可以获取多只股票的行情信息 #返回值:market,code,active1,price,last_close,open,high,low,reversed_bytes0,reversed_bytes1,vol,cur_vol,amount,s_vol, #reversed_bytes2,reversed_bytes3,bid1,ask1,bid_vol1,ask_vol1,bid2,ask2,bid_vol2,ask_vol2,bid3,ask3,bid_vol3,ask_vol3,bid4, #ask4,bid_vol4,ask_vol4,bid5,ask5,bid_vol5,ask_vol5,reversed_bytes4,reversed_bytes5,reversed_bytes6,reversed_bytes7, #reversed_bytes8,reversed_bytes9,active2 def get_security_quotes(self): df = self.api.to_df( self.api.get_security_quotes([(0, '000001'), (1, '600300')])) print(df) # 获取市场股票数量 #返回值:value def get_security_count(self): df = self.api.to_df(self.api.get_security_count(0)) #0 - 深圳, 1 - 上海 print(df) # 获取股票列表,返回值里面除了股票,还有国债等 #返回值:code,volunit,decimal_point,name,pre_close def get_security_list(self): df = self.api.to_df(self.api.get_security_list( 0, 10000)) # 市场代码, 起始位置 如: 0,0 或 1,100 print(df) # 获取指数k线 #输入参数同股票k线接口 # 返回值:open,close,high,low,vol,amount,year,month,day,hour,minute,datetime,up_count down_count def get_index_bars(self): index_dict_cn = { "上证指数": "999999", "深证成指": "399001", "中小板指": "399005", "创业板指": "399006", "深证综指": "399106", "上证50": "000016", "沪深300": "000300" } index_dict = { "sh": "999999", "sz": "399001", "zxb": "399005", "cyb": "399006", "szz": "399106", "sz50": "000016", "hs300": "000300" } for key in index_dict.keys(): df = self.api.to_df( self.api.get_index_bars(9, 1, index_dict[key], 0, 2)) print(df) # 查询分时行情,最近交易日的数据,一分钟一条记录 #返回值:price,vol def get_minute_time_data(self): df = self.api.to_df(self.api.get_minute_time_data( 1, '600300')) #市场代码, 股票代码 print(df) # 查询历史分时行情 # 返回值:price,vol def get_history_minute_time_data(self): df = self.api.to_df( self.api.get_history_minute_time_data(TDXParams.MARKET_SH, '603887', 20200420)) #市场代码, 股票代码,时间 print(df) # 查询分笔成交,最近交易日数据 #返回值:time,price,vol,num,buyorsell def get_transaction_data(self): df = self.api.to_df( self.api.get_transaction_data(TDXParams.MARKET_SZ, '000001', 0, 30)) #市场代码, 股票代码,起始位置, 数量 print(df) # 查询历史分笔成交 #返回值:time,price,vol,buyorsell def get_history_transaction_data(self): df = self.api.to_df( self.api.get_history_transaction_data( TDXParams.MARKET_SZ, '000001', 0, 10, 20170209)) #市场代码, 股票代码,起始位置,日期 数量 print(df) # 查询公司信息目录,返回的不是具体数据 #返回值:name,filename,start,length def get_company_info_category(self): df = self.api.to_df( self.api.get_company_info_category(TDXParams.MARKET_SZ, '000001')) #市场代码, 股票代码 print(df) # 读取公司信息详情 #返回值:value def get_company_info_content(self): df = self.api.to_df( self.api.get_company_info_content( 0, '000001', '000001.txt', 0, 1000)) #市场代码, 股票代码, 文件名, 起始位置, 数量 print(df) # 读取除权除息信息 #返回值:year,month,day,category,name,fenhong,peigujia,songzhuangu,peigu def get_xdxr_info(self): df = self.api.to_df(self.api.get_xdxr_info(1, '600300')) #市场代码, 股票代码 print(df) # 读取财务信息 #返回值:market,code,liutongguben,province,industry,updated_date,ipo_date,zongguben,guojiagu,faqirenfarengu,farengu,bgu,hgu,zhigonggu, #zongzichan,liudongzichan,gudingzichan,wuxingzichan,gudongrenshu,liudongfuzhai,changqifuzhai,zibengongjijin,jingzichan,zhuyingshouru, #zhuyinglirun,yingshouzhangkuan,yingyelirun,touzishouyu,jingyingxianjinliu,zongxianjinliu,cunhuo,lirunzonghe,shuihoulirun,jinglirun,weifenlirun,baoliu1,baoliu2 def get_finance_info(self): df = self.api.to_df(self.api.get_finance_info(1, '600300')) #市场代码, 股票代码 print(df) # 读取k线信息 # 返回值:value def get_k_data(self): df = self.api.to_df( self.api.get_k_data('600300', '2017-07-03', '2017-07-10')) #股票代码, 开始时间, 结束时间 print(df) # 读取板块信息 #返回值:blockname, block_type, code_index, code """ BLOCK_SZ = "block_zs.dat";BLOCK_FG = "block_fg.dat";BLOCK_GN = "block_gn.dat";BLOCK_DEFAULT = "block.dat" """ def get_and_parse_block_info(self): ##指数板块 风格板块 概念板块 一般板块 block_filename = [ "block_zs.dat", "block_fg.dat", "block_gn.dat", "block.dat" ] for block in block_filename: df = self.api.to_df( self.api.get_and_parse_block_info(block)) #板块文件名称 filename = config.tdx_csv_block + block[0:-4] + ".csv" if os.path.isfile(filename): os.remove(filename) df.to_csv(filename, index=False, mode='w', header=True, sep=',', encoding="utf_8_sig") else: df.to_csv(filename, index=False, mode='w', header=True, sep=',', encoding="utf_8_sig") # 读取板块信息,多个类型封装到一个df对象中返回 # 返回值:data_source, block_category, block_type, block_name, block_code, ts_code, create_time def update_block_member(self): ##指数板块 风格板块 概念板块 一般板块 #block_filename = ["block_zs.dat", "block_fg.dat", "block_gn.dat", "block.dat"] block_filename = ["block_zs.dat", "block_fg.dat", "block_gn.dat"] #block.dat中的数据都包含在其他版块里了,这个可以去掉 data_source = "tdx" dfall = None for block in block_filename: df = self.api.to_df( self.api.get_and_parse_block_info(block)) # 板块文件名称 df['data_source'] = data_source if block == "block.dat": df['block_category'] = data_source + ".yb" else: df['block_category'] = data_source + "." + block[6:8] df['block_type'] = df['block_type'].map(lambda x: str(x)) df['block_type'] = df['block_category'].str.cat( df['block_type'], sep=".") #, sep = "." df['block_code'] = "" #使用pd直接插入到数据库时,字段不能是None值 df['ts_code'] = df['code'].apply(lambda x: x + ".SH" if x[0:1] == "6" else x + ".SZ") if (dfall is not None) and (not dfall.empty): dfall = dfall.append(df, ignore_index=True) else: dfall = df if (dfall is None) or (dfall.empty): return None dfall.rename(columns={'blockname': 'block_name'}, inplace=True) dfall['create_time'] = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time())) dfall = dfall[[ 'data_source', 'block_category', 'block_type', 'block_name', 'block_code', 'ts_code', 'create_time' ]] #列重排序 #分组统计 dfg = dfall.groupby(by=[ 'data_source', 'block_category', 'block_type', 'block_name', 'block_code' ], as_index=False).count() # 分组求每组数量 dfg.rename(columns={'ts_code': 'member_count'}, inplace=True) #ts_code列数值为汇总值,需要重命名 dfg['create_time'] = time.strftime( '%Y-%m-%d %H:%M:%S', time.localtime(time.time())) #create_time列数值为汇总值,需要重新赋值 delete_condition = f"data_source = '{data_source}'" mysql_script.df2db_update(delete_condition=delete_condition, block_basic_df=dfg, block_member_df=dfall) return (len(dfg), len(dfall)) #获取一段时间的1分钟数据,因为每次调用接口只能返回3天的分钟数据(240*3),需要分多次调用 #返回值:0没有提取到数据;1提取到数据 def get_minute1_data(self, category, market, code, start_date, end_date): init_start_date = start_date.replace('-', '') init_end_date = end_date.replace('-', '') day = datatime_util.diffrentPeriod(datatime_util.DAILY, start_date, end_date) df = self.get_security_bars_minute1(category, market, code, 0, 240 * 3) # 返回DataFrame if df is None or df.empty: print('{0}没有交易数据'.format(code)) return 0 print(market, '--', code, '--', start_date, '--', end_date) #print("最大值:",df.groupby('datetime').max()) #print(df.describe()) #df数据统计 data_start_date = df.min()['trade_date'] data_end_date = df.max()['trade_date'] start_date = start_date.replace('-', '') end_date = end_date.replace('-', '') if data_end_date < start_date or end_date < data_start_date: print("采集时间在数据范围之外,退出函数") return 0 elif end_date > data_end_date: end_date = data_end_date if start_date < data_start_date: #最近三天的数据中,去掉无用的数据后即是最终数据 #需要取的数据还有三天前的数据,需要继续向前取 n = (day - 3) // 3 m = (day - 3) % 3 for i in range(0, n): dfn = self.get_security_bars_minute1(category, market, code, 240 * 3 * (i + 1), 240 * 3) # 返回DataFrame if (dfn is not None) and (not dfn.empty): df = dfn.append(df, ignore_index=True) if m > 0: dfn = self.get_security_bars_minute1(category, market, code, 240 * 3 * (n + 1), 240 * m) if (dfn is not None) and (not dfn.empty): df = dfn.append(df, ignore_index=True) df = df.sort_values(by=['trade_date', 'time_index'], axis=0, ascending=True) #过滤掉start_date, end_date之外的数据 df = df[(df['trade_date'] >= str(init_start_date)) & (df['trade_date'] <= str(init_end_date))] #每个条件要用括号()括起来 dict = {0: 'SZ', 1: 'SH'} ts_code = code + "." + dict[market] filename = config.tdx_csv_minline1_all + ts_code + ".csv" if os.path.isfile(filename): df.to_csv(filename, index=False, mode='a', header=False, sep=',', encoding="utf_8_sig") print("更新一分钟all股票数据:", filename) else: df.to_csv(filename, index=False, mode='w', header=True, sep=',', encoding="utf_8_sig") print("新增加的一分钟all股票数据:", filename)
print("下载完成,正在解压...") os.system('unrar x -o+ -inul {} {}'.format(dest_filename, dest_dir)) print("解压完成,正在导入...") add_count = qianlong_import_weight(connect, dest_dir + '/weight', 'SH') add_count += qianlong_import_weight(connect, dest_dir + '/weight', 'SZ') print("导入数量:", add_count) """ print("\n导入上证分笔数据") #add_count = import_trans(connect, 'SH', quotations, api, dest_dir, max_days=2, progress=ProgressBar) print("\n导入数量:", add_count) print("\n导入上证分时数据") add_count = import_time(connect, 'SZ', quotations, api, dest_dir, max_days=3, progress=ProgressBar) print("\n导入数量:", add_count) api.disconnect() connect.close() endtime = time.time() print("\nTotal time:") print("%.2fs" % (endtime - starttime)) print("%.2fm" % ((endtime - starttime) / 60))
class SP(object): def __init__(self, userid="account1", number=2, mock=True, server="http://192.168.118.1:65000"): ''' 根据实际账户资金,以股指1手为基本对冲单位进行配置。 ''' self.products = { '000905': { 'args': (3, 6, 65), 'stocklst': { "510500": 0 } } } self.datatype = 0 self.userid = userid self.api = TdxHq_API(heartbeat=True) self.trader = None self.trading = False self.mock = mock self.permoney = 200 #指数一个点对应的金额 self.number = number #指数的倍数 self.server = server self.TDX_IP_SETS = STOCK_IP_SETS if mock: self.judgerate = 100 else: self.judgerate = 10 def connect(self): for ip in self.TDX_IP_SETS: try: if self.api.connect(ip, 7709): return except: pass def getdata(self, product, market=1, number=10000, pn=400): data = [] for i in range(int(number / pn) + 1): temp = self.api.get_index_bars(self.datatype, market, product, (int(number / pn) - i) * pn, pn) if not temp or len(temp) < pn: for _ in range(2): temp = self.api.get_index_bars(self.datatype, market, product, (int(number / pn) - i) * pn, pn) if not temp or len(temp) < pn: logger.info("record not reach the limit!") else: break try: data += temp except: self.connect() df = self.api.to_df(data)[["open", "close", "high", "low", "datetime"]] df.set_index("datetime", inplace=True, drop=False) return df def disconnect(self): logger.info("[DISCONNECT]:statrt disconnect!!!!!") if self.istradeday: self.api.disconnect() logger.info("[DISCONNECT]:disconnect finished!!!!!") def set_number(self): '''设置日内交易的单次手数 ''' price = self.api.get_security_quotes([(1, '510500')])[0]["last_close"] index_price = self.api.get_security_quotes([(1, '000905') ])[0]["last_close"] self.products["000905"]["stocklst"]["510500"] = int( index_price * self.number * self.permoney / price / self.judgerate) * 100 def judgetradeday(self, ): today = datetime.datetime.today().date().strftime('%Y-%m-%d') df = ts.trade_cal() return df[(df["calendarDate"] == today)].isOpen.values[0] @property def istradeday(self): if not hasattr(self, "_istradeday"): self._istradeday = self.judgetradeday() return self._istradeday def initial(self): '''每天初始化设置 ''' logger.info("[INITIAL]:start initial !!!!") if not self.istradeday: self.trading = False return self.trading = True logger.info("[INITIAL]:try to create connect... ") self.connect() self.trader = trade(UserID=self.userid, api=self.api, mock=self.mock, server=self.server) logger.info("[INITIAL]:connect successful!") self.set_number() #设置手数 logger.info("[INITIAL]:initial finished !!!!!") def handledata(self, df, args=[]): df.loc[:, "number"] = range(df.shape[0]) s, m, l = args for i in args: #5 15 60 D key = str(5 * i) df.loc[:, key + "high"] = df["high"].rolling(i).max() df.loc[:, key + "low"] = df["low"].rolling(i).min() df.loc[:, key + "atr"] = (df[key + "high"] - df[key + "low"]).rolling(10 * i).mean() df.loc[:, key + "med"] = (df[key + "high"] + df[key + "low"]) / 2 df.loc[:, key + "HH"] = df[key + "med"] + 1.5 * df[key + "atr"] df.loc[:, key + "LL"] = df[key + "med"] - 1.5 * df[key + "atr"] df.loc[:, key + 'HHmax'] = df[key + 'HH'].rolling(10 * i).max() df.loc[:, key + 'LLmin'] = df[key + 'LL'].rolling(10 * i).min() df.loc[df[key + 'HH'] >= df[key + 'HHmax'], key + 'hmark'] = df["number"] df.loc[df[key + 'LL'] <= df[key + 'LLmin'], key + 'lmark'] = df["number"] df[key + 'hmark'].fillna(method="ffill", inplace=True) df[key + 'hmark'].fillna(0, inplace=True) df[key + 'lmark'].fillna(method="ffill", inplace=True) df[key + 'lmark'].fillna(0, inplace=True) df.loc[:, key + 'UP'] = df[key + 'hmark'] >= df[key + 'lmark'] # debuginfo.append({key+'hmark':df.iloc[-1][key+'hmark'],key+'lmark':df.iloc[-1][key+'lmark']}) df.fillna(method="ffill", inplace=True) df.dropna(inplace=True) # logger.info("trademessage:{}".format(debuginfo)) result = (df.iloc[-1][str(5 * l) + "UP"] > 0) & (df.iloc[-1][str(5 * s) + "UP"] > 0) result |= (df.iloc[-1][str(5 * l) + "UP"] > 0) & (df.iloc[-1][str(5 * m) + "UP"] > 0) result |= (df.iloc[-1][str(5 * l) + "UP"] <= 0) & (df.iloc[-1][ str(5 * m) + "UP"] > 0) & (df.iloc[-1][str(5 * s) + "UP"] > 0) return result def sync(self, idx, director=True): stocks = self.products[idx]["stocklst"] for stock, number in stocks.items(): if not director: number = 2 * number #空信号,清仓 else: number = 4 * number #判断现有持仓 try: h_number = self.hd_df.ix[stock]["证券数量"] except: h_number = 0 #补仓差 cangcha = int((number - h_number) / 100) * 100 if h_number > 0 and abs( cangcha ) / number < 0.05: #如果有持仓,同时仓差小于5% 不进行更改,为了处理频繁加减仓达到问题 return if cangcha > 0: logger.info("[RUN]:buy code:{}, number:{}".format( stock, number - h_number)) self.buy(stock, cangcha) elif cangcha < 0: logger.info("[RUN]:sell code:{}, number:{}".format( stock, h_number - number)) lastholdnumber = self.hd_df.ix[stock]["可卖数量"] self.sell(stock, min(-cangcha, lastholdnumber)) def buy(self, stock, number, jump=0.02): self.trader.buy(stock, number, jump=jump) def sell(self, stock, number, jump=0.02): self.trader.sell(stock, number, jump=jump) def check_position(self, status): '''检查仓位情况 ''' self.handleposition() handlelist = [] for ins, v in status.items(): if ins not in self.g_df.index: handlelist.append(ins) elif self.g_df.ix[ins]["Position"] != v["number"]: handlelist.append(ins) return handlelist def handleposition(self): '''计算多仓,空仓,以及昨仓和今仓 ''' self.trader.cancelorder() #先尝试撤单 _, holdlists = self.trader.position() self.hd_df = holdlists if holdlists.shape[0] > 0: self.hd_df.set_index("证券代码", inplace=True) self.hd_df.index = self.hd_df.index.astype(np.str) return self.hd_df def run(self): logger.info("[RUN]:start run !!!!!") if not self.trading: return self.handleposition() rst = {} for idx in list(self.products.keys()): director = self.handledata(self.getdata(idx, market=1), self.products[idx]["args"]) #用指数出信号 logger.info("[RUN]:trademessage: block:{}, director:{}".format( idx, director)) self.sync(idx, director) rst[idx] = { "up": director, "number": self.products[idx]["stocklst"], "product": idx } logger.info("[RUN]:lastest position status:{}".format(rst)) logger.info("[RUN]:run finished !!!!!") def nhg(self): '''尾盘自动进行逆回购 ''' if not self.mock: self.trader.autobuy()
class PYTDXService(): """pytdx数据服务类""" def __init__(self): """Constructor""" self.connected = False # 数据服务连接状态 self.hq_api = None # 行情API def connect_api(self): """连接API""" # 连接增强行情API并检查连接情况 try: if not self.connected: host = SETTINGS["TDX_HOST"] port = SETTINGS["TDX_PORT"] self.hq_api = TdxHq_API() self.hq_api.connect(host, port) self.connected = True return True except Exception: raise ConnectionError("pytdx连接错误") def get_realtime_data(self, symbol: str): """获取股票实时数据""" try: symbols = self.generate_symbols(symbol) df = self.hq_api.to_df(self.hq_api.get_security_quotes(symbols)) return df except Exception: raise ValueError("股票数据获取失败") def get_history_transaction_data(self, symbol, date): """ 查询历史分笔数据 get_history_transaction_data(TDXParams.MARKET_SZ, '000001', 0, 10, 20170209) 参数:市场代码, 股票代码, 起始位置, 数量, 日期 输出[time, price, vol, buyorsell(0:buy, 1:sell, 2:平)] """ # 获得标的 code, market = self.check_symbol(symbol) # 设置参数 check_date = int(date) count = 2000 data_list = [] position = [6000, 4000, 2000, 0] for start in position: data = self.hq_api.to_df( self.hq_api.get_history_transaction_data( market, code, start, count, check_date)) data_list.append(data) df = pd.concat(data_list) df.drop_duplicates(inplace=True) return df @staticmethod def generate_symbols(symbol: str): """组装symbols数据,pytdx接收的是以市场代码和标的代码组成的元祖的list""" new_symbols = [] code, exchange = symbol.split('.') new_symbols.append((exchange_map[exchange], code)) return new_symbols @staticmethod def check_symbol(symbol: str): """检查标的格式""" if symbol: code, market = symbol.split('.') market = exchange_map.get(market) return code, market else: return False def close(self): """数据服务关闭""" self.connected = False self.hq_api.disconnect()
class StockMod(object): def __init__(self,debug=False): # 初始化tdx self.debug = debug oplogs("Loading stock mod...") self.api = TdxHq_API(heartbeat=(not debug),auto_retry=True) self.connect_to_tdx() self.load_stocklist() self.monitor_queue = dict() def __del__(self): oplogs("Unloading stock mod...") self.api.disconnect() #self.stop_jobs() def connect_to_tdx(self): random.shuffle(HZ_HOST_IPS) for ip in HZ_HOST_IPS: if self.api.connect(ip, 7709): oplogs("Connected to %s success" %ip) return else: oplogs("Connected to %s failed!!" %ip) def load_stocklist(self): try: oplogs("get_stock_basics() ...") self.stocklist = ts.get_stock_basics() oplogs("get_stock_basics() load from remote success") except: # 如果加载远程股票列表失败, 从本地列表读取 oplogs("get_stock_basics() timeout, use local cached file instead") cur_path=os.path.dirname(os.path.realpath(__file__)) stocklist_path=os.path.join(cur_path,'all.csv') df = pd.read_csv(stocklist_path, dtype={'code':'object'},encoding='GBK') df = df.set_index('code') self.stocklist = df # remove space in stock name self.stocklist['name'] = self.stocklist['name'].str.replace(' ','') self.stocklist.drop(self.stocklist[self.stocklist.name == '机器人'].index, inplace=True) df_index = pd.DataFrame([['999999','上证指数'], ['399001','深证成指'], ['399006','创业板指']], index=['999999','399001','399006'], columns=['code', 'name']) self.stocklist = self.stocklist.append(df_index) # create dict and list to shorten search time self.stockdict = self.stocklist.to_dict()["name"] self.stockrdict = {value:key for key,value in self.stockdict.items()} self.stockcode = self.stocklist.index.to_list() self.stockname = self.stocklist.name.to_list() def get_market(self, code): # 计算市场代码,1=sh, 0=sz market = 0 if code == '999999': return 1 elif code in ['399001','399006']: return 0 elif code[0] == '6': market = 1 return market def get_stock_price(self, code): if code not in self.stockcode: return "" market = self.get_market(code) oplogs("get_stock_price called,code=%s,market=%s" %(code,market)) price = self.api.get_security_quotes([(market,code)]) last_close = price[0]['last_close'] current_price = price[0]['price'] if last_close >0: change = (current_price/last_close - 1)*100 else: change = 0 result = "[%s]%.2f 涨幅:%.2f%%" %(self.stockdict[code],current_price,change ) return result def get_stock_price_by_name(self, stockname): if stockname in self.stockname: return self.get_stock_price(self.stockrdict[stockname]) return "" def get_fast_alarms(self): pass def scan_stock(self, group, msg): info = msg.lower() for code in self.stockcode: if code in info: self.add_to_monitor_group(group, code) for name in self.stockname: if name in info: return self.add_to_monitor_group(group, self.stockrdict[name]) def isvalid_stock(self, stock): if stock.isdigit(): return stock in self.stockcode else: return stock in self.stockname def extract_stock_from_msg(self, msg): stocklist = [] for code in self.stockcode: if code in msg: stocklist.append(code) for name in self.stockname: if name in msg: stocklist.append(self.stockrdict[name]) return stocklist def add_to_monitor_group(self,group,stock): if stock.isdigit(): code = stock else: code = self.stockrdict[stock] if group in self.monitor_queue.keys(): if code not in self.monitor_queue[group]: self.monitor_queue[group].append(code) else: self.monitor_queue[group] = [code] oplogs("add_to_monitor_group called:%s %s" %(group, stock)) def get_group_stock_price(self, group): if group in self.monitor_queue.keys(): stocks = [] for code in self.monitor_queue[group]: price, change = self.get_stock_daily(code) stocks.append([code,price,change]) sorted_alarms = sorted(stocks, key=lambda x: x[2],reverse=True) result = "本群关注:\n" for record in sorted_alarms: if record[1] == 0: result += "[%s]停盘\n" %self.stockdict[record[0]] else: newprice = "%.2f" %record[1] newchange = "%.2f%%" %record[2] result += "[%s]%s:%s\n" %(self.stockdict[record[0]], newprice, newchange) return result return "" def get_stock_daily(self, code): # 计算当日涨跌 market = self.get_market(code) data_day = self.api.get_security_quotes([(market,code)]) # get daily changes if self.debug: oplogs("get_stock_daily called") #print(self.api.to_df(data_day).ix[:,0:8]) last_close = data_day[0]['last_close'] current_price = data_day[0]['price'] if last_close >0: #正常情况 change = (current_price/last_close - 1)*100 else: change = 0 return current_price, change def get_stock_min(self, code, min=5): market = self.get_market(code) data_min = self.api.get_security_bars(0,market, code, 0, 1) # 5 min K bar if self.debug: oplogs("get_stock_min called") #print(self.api.to_df(data_min).ix[:,0:8]) open = data_min[0]['open'] close = data_min[0]['close'] high = data_min[0]['high'] low = data_min[0]['low'] if low >0 and open>0: # 波动范围 change_high_low = (high-low)/low*100 # 涨跌幅 change_close_open = (close-open)/open*100 else: change_high_low = 0 change_close_open = 0 return change_high_low, change_close_open def get_alarms(self,valve=3.0): # timeslice , slice in time series, default =5 min # range, monitor change range, default = 3%. Actual change range = abs((high-low)/low) groupalarms = dict() for (key, value) in self.monitor_queue.items(): # 获取单个group监控的股票报警 alarms = [] for code in value: # 当日涨跌 price, change = self.get_stock_daily(code) if price <= 0: # 停盘不监控 continue # 5分钟涨跌 highlow, closeopen = self.get_stock_min(code) # 判断波动幅度是否超过阈值 if abs(highlow) >=valve: alarm = [self.stockdict[code],price,change,highlow,closeopen] alarms.append(alarm) # 按5分钟涨跌幅排序 sorted_alarms = sorted(alarms, key=lambda x: (x[4],x[2]),reverse=True) # 打包所有group的报警列表 groupalarms[key] = sorted_alarms return groupalarms def del_from_list(self, group, stock): # group, group name # stock, stock name # remove stock from group if stock.isdigit(): code = stock else: code = self.stockrdict[stock] if group in self.monitor_queue.keys(): if code in self.monitor_queue[group]: self.monitor_queue[group].remove(code) oplogs("del_from_list called:%s %s" %(group, stock)) def test(self): data = self.api.get_security_bars(0,1, '600477', 0, 48) # 5 min K bar df = self.api.to_df(data) df['change_high_low'] = (df['high']-df['low'])/df['low']*100 df['change_close_open'] = (df['close']-df['open'])/df['open']*100 print(df)