def __init__(self, start_dt: Union[date, datetime], end_dt: Union[date, datetime]) -> None: """ 创建天勤回测类 Args: start_dt (date/datetime): 回测起始时间, 如果类型为 date 则指的是交易日, 如果为 datetime 则指的是具体时间点 end_dt (date/datetime): 回测结束时间, 如果类型为 date 则指的是交易日, 如果为 datetime 则指的是具体时间点 """ if isinstance(start_dt, datetime): self._start_dt = int(start_dt.timestamp() * 1e9) elif isinstance(start_dt, date): self._start_dt = _get_trading_day_start_time( int(datetime(start_dt.year, start_dt.month, start_dt.day).timestamp()) * 1000000000) else: raise Exception("回测起始时间(start_dt)类型 %s 错误, 请检查 start_dt 数据类型是否填写正确" % (type(start_dt))) if isinstance(end_dt, datetime): self._end_dt = int(end_dt.timestamp() * 1e9) elif isinstance(end_dt, date): self._end_dt = _get_trading_day_end_time( int(datetime(end_dt.year, end_dt.month, end_dt.day).timestamp()) * 1000000000) else: raise Exception("回测结束时间(end_dt)类型 %s 错误, 请检查 end_dt 数据类型是否填写正确" % (type(end_dt))) self._current_dt = self._start_dt # 记录当前的交易日 开始时间/结束时间 self._trading_day = _get_trading_day_from_timestamp(self._current_dt) self._trading_day_start = _get_trading_day_start_time(self._trading_day) self._trading_day_end = _get_trading_day_end_time(self._trading_day)
async def _send_diff(self): """发送数据到 api, 如果 self._diffs 不为空则发送 self._diffs, 不推进行情时间, 否则将时间推进一格, 并发送对应的行情""" if self._pending_peek: if not self._diffs: quotes = await self._generator_diffs(False) else: quotes = await self._generator_diffs(True) for ins, diff in quotes.items(): self._quotes[ins]["sended_init_quote"] = True for d in diff: self._diffs.append({ "quotes": { ins: d } }) if self._diffs: # 发送数据集中添加 backtest 字段,开始时间、结束时间、当前时间,表示当前行情推进是由 backtest 推进 if self._is_first_send: self._diffs.append({ "_tqsdk_backtest": { "start_dt": self._start_dt, "current_dt": self._current_dt, "end_dt": self._end_dt } }) self._is_first_send = False else: self._diffs.append({ "_tqsdk_backtest": { "current_dt": self._current_dt } }) # 切换交易日,将历史的主连合约信息添加的 diffs if self._current_dt > self._trading_day_end: # 使用交易日结束时间,每个交易日切换只需要计算一次交易日结束时间 # 相比发送 diffs 前每次都用 _current_dt 计算当前交易日,计算次数更少 self._trading_day_start = _get_trading_day_start_time(_get_trading_day_from_timestamp(self._current_dt)) self._trading_day_end = _get_trading_day_end_time(_get_trading_day_from_timestamp(self._current_dt)) self._diffs.append({ "quotes": self._get_history_cont_quotes( datetime.fromtimestamp(self._trading_day_end / 1e9).strftime("%Y%m%d") ) }) self._diffs.append({ "quotes": {k: {'expired': v.get('expire_datetime', float('nan')) <= self._trading_day_start} for k, v in self._data.get('quotes').items()} }) self._sim_recv_chan_send_count += 1 if self._sim_recv_chan_send_count > 10000: self._sim_recv_chan_send_count = 0 self._diffs.append(self._gc_data()) rtn_data = { "aid": "rtn_data", "data": self._diffs, } self._diffs = [] self._pending_peek = False await self._sim_recv_chan.send(rtn_data)
def _get_trading_timestamp(quote, current_datetime: str): """ 将 quote 在 current_datetime 所在交易日的所有可交易时间段转换为纳秒时间戳(tqsdk内部使用的时间戳统一为纳秒)并返回 """ # 获取当前交易日时间戳 current_trading_day_timestamp = _get_trading_day_from_timestamp( int(datetime.datetime.strptime(current_datetime, "%Y-%m-%d %H:%M:%S.%f").timestamp() * 1e6) * 1000) # 获取上一交易日时间戳 last_trading_day_timestamp = _get_trading_day_from_timestamp( _get_trading_day_start_time(current_trading_day_timestamp) - 1) trading_timestamp = { "day": TqSim._get_period_timestamp(current_trading_day_timestamp, quote["trading_time"].get("day", [])), "night": TqSim._get_period_timestamp(last_trading_day_timestamp, quote["trading_time"].get("night", [])) } return trading_timestamp
def dt_func (self): # 回测和复盘模式,用 _api._account 一定是 TqSim, 使用 TqSim _get_current_timestamp() 提供的时间 # todo: 使用 TqSim.EPOCH if self._data["action"]["mode"] == "backtest": return self._data['_tqsdk_backtest']['current_dt'] elif self._data["action"]["mode"] == "replay": tqsim_current_timestamp = self._api._account._account_list[0]._get_current_timestamp() if tqsim_current_timestamp == 631123200000000000: # 未收到任何行情, TqSim 时间没有更新 return _get_trading_day_start_time(self._data['_tqsdk_replay']['replay_dt']) else: return tqsim_current_timestamp else: return int(datetime.now().timestamp() * 1e9)
def __init__(self, api: TqApi, symbol_list: Union[str, List[str]], dur_sec: int, start_dt: Union[date, datetime], end_dt: Union[date, datetime], csv_file_name: str) -> None: """ 创建历史数据下载器实例 Args: api (TqApi): TqApi实例,该下载器将使用指定的api下载数据 symbol_list (str/list of str): 需要下载数据的合约代码,当指定多个合约代码时将其他合约按第一个合约的交易时间对齐 dur_sec (int): 数据周期,以秒为单位。例如: 1分钟线为60,1小时线为3600,日线为86400,Tick数据为0 start_dt (date/datetime): 起始时间, 如果类型为 date 则指的是交易日, 如果为 datetime 则指的是具体时间点 end_dt (date/datetime): 结束时间, 如果类型为 date 则指的是交易日, 如果为 datetime 则指的是具体时间点 csv_file_name (str): 输出csv的文件名 Example:: from datetime import datetime, date from contextlib import closing from tqsdk import TqApi, TqSim from tqsdk.tools import DataDownloader api = TqApi(TqSim()) download_tasks = {} # 下载从 2018-01-01 到 2018-09-01 的 SR901 日线数据 download_tasks["SR_daily"] = DataDownloader(api, symbol_list="CZCE.SR901", dur_sec=24*60*60, start_dt=date(2018, 1, 1), end_dt=date(2018, 9, 1), csv_file_name="SR901_daily.csv") # 下载从 2017-01-01 到 2018-09-01 的 rb主连 5分钟线数据 download_tasks["rb_5min"] = DataDownloader(api, symbol_list="*****@*****.**", dur_sec=5*60, start_dt=date(2017, 1, 1), end_dt=date(2018, 9, 1), csv_file_name="rb_5min.csv") # 下载从 2018-01-01凌晨6点 到 2018-06-01下午4点 的 cu1805,cu1807,IC1803 分钟线数据,所有数据按 cu1805 的时间对齐 # 例如 cu1805 夜盘交易时段, IC1803 的各项数据为 N/A # 例如 cu1805 13:00-13:30 不交易, 因此 IC1803 在 13:00-13:30 之间的K线数据会被跳过 download_tasks["cu_min"] = DataDownloader(api, symbol_list=["SHFE.cu1805", "SHFE.cu1807", "CFFEX.IC1803"], dur_sec=60, start_dt=datetime(2018, 1, 1, 6, 0 ,0), end_dt=datetime(2018, 6, 1, 16, 0, 0), csv_file_name="cu_min.csv") # 下载从 2018-05-01凌晨0点 到 2018-06-01凌晨0点 的 T1809 盘口Tick数据 download_tasks["T_tick"] = DataDownloader(api, symbol_list=["CFFEX.T1809"], dur_sec=0, start_dt=datetime(2018, 5, 1), end_dt=datetime(2018, 6, 1), csv_file_name="T1809_tick.csv") # 使用with closing机制确保下载完成后释放对应的资源 with closing(api): while not all([v.is_finished() for v in download_tasks.values()]): api.wait_update() print("progress: ", { k:("%.2f%%" % v.get_progress()) for k,v in download_tasks.items() }) """ self._api = api if isinstance(start_dt, datetime): self._start_dt_nano = int(start_dt.timestamp() * 1e9) else: self._start_dt_nano = _get_trading_day_start_time(int(datetime(start_dt.year, start_dt.month, start_dt.day).timestamp()) * 1000000000) if isinstance(end_dt, datetime): self._end_dt_nano = int(end_dt.timestamp() * 1e9) else: self._end_dt_nano = _get_trading_day_end_time(int(datetime(end_dt.year, end_dt.month, end_dt.day).timestamp()) * 1000000000) self._current_dt_nano = self._start_dt_nano self._symbol_list = symbol_list if isinstance(symbol_list, list) else [symbol_list] self._dur_nano = dur_sec * 1000000000 if self._dur_nano == 0 and len(self._symbol_list) != 1: raise Exception("Tick序列不支持多合约") self._csv_file_name = csv_file_name self._task = self._api.create_task(self._download_data())
async def _gen_serial(self, ins, dur): """k线/tick 序列的 async generator, yield 出来的行情数据带有时间戳, 因此 _send_diff 可以据此归并""" # 先定位左端点, focus_datetime 是 lower_bound ,这里需要的是 upper_bound # 因此将 view_width 和 focus_position 设置成一样,这样 focus_datetime 所对应的 k线刚好位于屏幕外 chart_info = { "aid": "set_chart", "chart_id": _generate_uuid("PYSDK_backtest"), "ins_list": ins, "duration": dur, "view_width": 8964, # 设为8964原因:可满足用户所有的订阅长度,并在backtest中将所有的 相同合约及周期 的K线用同一个serial存储 "focus_datetime": int(self._current_dt), "focus_position": 8964, } chart = _get_obj(self._data, ["charts", chart_info["chart_id"]]) current_id = None # 当前数据指针 serial = _get_obj( self._data, ["klines", ins, str(dur)] if dur != 0 else ["ticks", ins]) async with TqChan(self._api, last_only=True) as update_chan: serial["_listener"].add(update_chan) chart["_listener"].add(update_chan) await self._md_send_chan.send(chart_info.copy()) try: async for _ in update_chan: if not (chart_info.items() <= _get_obj(chart, ["state"]).items()): # 当前请求还没收齐回应, 不应继续处理 continue left_id = chart.get("left_id", -1) right_id = chart.get("right_id", -1) last_id = serial.get("last_id", -1) if (left_id == -1 and right_id == -1) or last_id == -1: # 定位信息还没收到, 或数据序列还没收到 continue if self._data.get("mdhis_more_data", True): self._data["_listener"].add(update_chan) continue else: self._data["_listener"].discard(update_chan) if current_id is None: current_id = max(left_id, 0) while True: if current_id > last_id: # 当前 id 已超过 last_id return if current_id - chart_info.get("left_kline_id", left_id) > 5000: # 当前 id 已超出订阅范围, 需重新订阅后续数据 chart_info["left_kline_id"] = current_id chart_info.pop("focus_datetime", None) chart_info.pop("focus_position", None) await self._md_send_chan.send(chart_info.copy()) # 将订阅的8964长度的窗口中的数据都遍历完后,退出循环,然后再次进入并处理下一窗口数据 # (因为在处理过5000条数据的同时向服务器订阅从当前id开始的新一窗口的数据,在当前窗口剩下的3000条数据处理完后,下一窗口数据也已经收到) if current_id > right_id: break item = { k: v for k, v in serial["data"].get( str(current_id), {}).items() } if dur == 0: diff = { "ticks": { ins: { "last_id": current_id, "data": { str(current_id): item, str(current_id - 8964): None, } } } } if item["datetime"] > self._end_dt: # 超过结束时间 return yield item[ "datetime"], diff, self._get_quotes_from_tick( item) else: diff = { "klines": { ins: { str(dur): { "last_id": current_id, "data": { str(current_id): { "datetime": item["datetime"], "open": item["open"], "high": item["open"], "low": item["open"], "close": item["open"], "volume": 0, "open_oi": item["open_oi"], "close_oi": item["open_oi"], }, str(current_id - 8964): None, } } } } } timestamp = item[ "datetime"] if dur < 86400000000000 else _get_trading_day_start_time( item["datetime"]) if timestamp > self._end_dt: # 超过结束时间 return yield timestamp, diff, self._get_quotes_from_kline_open( self._data["quotes"][ins], timestamp, item) # K线刚生成时的数据都为开盘价 diff = { "klines": { ins: { str(dur): { "data": { str(current_id): item, } } } } } timestamp = item[ "datetime"] + dur - 1000 if dur < 86400000000000 else _get_trading_day_end_time( item["datetime"]) - 999 if timestamp > self._end_dt: # 超过结束时间 return yield timestamp, diff, self._get_quotes_from_kline( self._data["quotes"][ins], timestamp, item) # K线结束时生成quote数据 current_id += 1 finally: # 释放chart资源 chart_info["ins_list"] = "" await self._md_send_chan.send(chart_info.copy())
async def _gen_serial(self, ins, dur): """k线/tick 序列的 async generator, yield 出来的行情数据带有时间戳, 因此 _send_diff 可以据此归并""" # 先定位左端点, focus_datetime 是 lower_bound ,这里需要的是 upper_bound # 因此将 view_width 和 focus_position 设置成一样,这样 focus_datetime 所对应的 k线刚好位于屏幕外 # 使用两个长度为 8964 的 chart,去缓存/回收下游需要的数据 chart_id_a = _generate_uuid("PYSDK_backtest") chart_id_b = _generate_uuid("PYSDK_backtest") chart_info = { "aid": "set_chart", "chart_id": chart_id_a, "ins_list": ins, "duration": dur, "view_width": 8964, # 设为8964原因:可满足用户所有的订阅长度,并在backtest中将所有的 相同合约及周期 的K线用同一个serial存储 "focus_datetime": int(self._current_dt), "focus_position": 8964, } chart_a = _get_obj(self._data, ["charts", chart_id_a]) chart_b = _get_obj(self._data, ["charts", chart_id_b]) symbol_list = ins.split(',') current_id = None # 当前数据指针 if dur == 0: serials = [_get_obj(self._data, ["ticks", symbol_list[0]])] else: serials = [_get_obj(self._data, ["klines", s, str(dur)]) for s in symbol_list] async with TqChan(self._api, last_only=True) as update_chan: for serial in serials: serial["_listener"].add(update_chan) chart_a["_listener"].add(update_chan) chart_b["_listener"].add(update_chan) await self._md_send_chan.send(chart_info.copy()) try: async for _ in update_chan: chart = _get_obj(self._data, ["charts", chart_info["chart_id"]]) if not (chart_info.items() <= _get_obj(chart, ["state"]).items()): # 当前请求还没收齐回应, 不应继续处理 continue left_id = chart.get("left_id", -1) right_id = chart.get("right_id", -1) if (left_id == -1 and right_id == -1) or chart.get("more_data", True): continue # 定位信息还没收到, 数据没有完全收到 last_id = serials[0].get("last_id", -1) if last_id == -1: continue # 数据序列还没收到 if self._data.get("mdhis_more_data", True): self._data["_listener"].add(update_chan) continue else: self._data["_listener"].discard(update_chan) if current_id is None: current_id = max(left_id, 0) # 发送下一段 chart 8964 根 kline chart_info["chart_id"] = chart_id_b if chart_info["chart_id"] == chart_id_a else chart_id_a chart_info["left_kline_id"] = right_id chart_info.pop("focus_datetime", None) chart_info.pop("focus_position", None) await self._md_send_chan.send(chart_info.copy()) while True: if current_id > last_id: # 当前 id 已超过 last_id return # 将订阅的8964长度的窗口中的数据都遍历完后,退出循环,然后再次进入并处理下一窗口数据 if current_id > right_id: break item = {k: v for k, v in serials[0]["data"].get(str(current_id), {}).items()} if dur == 0: diff = { "ticks": { ins: { "last_id": current_id, "data": { str(current_id): item, str(current_id - 8964): None, } } } } if item["datetime"] > self._end_dt: # 超过结束时间 return yield item["datetime"], diff, self._get_quotes_from_tick(item) else: timestamp = item["datetime"] if dur < 86400000000000 else _get_trading_day_start_time( item["datetime"]) if timestamp > self._end_dt: # 超过结束时间 return binding = serials[0].get("binding", {}) diff = { "klines": { symbol_list[0]: { str(dur): { "last_id": current_id, "data": { str(current_id): { "datetime": item["datetime"], "open": item["open"], "high": item["open"], "low": item["open"], "close": item["open"], "volume": 0, "open_oi": item["open_oi"], "close_oi": item["open_oi"], } } } } } } for chart_id in self._serials[(ins, dur)]["chart_id_set"]: diff["charts"] = { chart_id: { "right_id": current_id # api 中处理多合约 kline 需要 right_id 信息 } } for i, symbol in enumerate(symbol_list): if i == 0: diff_binding = diff["klines"][symbol_list[0]][str(dur)].setdefault("binding", {}) continue other_id = binding.get(symbol, {}).get(str(current_id), -1) if other_id >= 0: diff_binding[symbol] = {str(current_id): str(other_id)} other_item = serials[i]["data"].get(str(other_id), {}) diff["klines"][symbol] = { str(dur): { "last_id": other_id, "data": { str(other_id): { "datetime": other_item["datetime"], "open": other_item["open"], "high": other_item["open"], "low": other_item["open"], "close": other_item["open"], "volume": 0, "open_oi": other_item["open_oi"], "close_oi": other_item["open_oi"], } } } } yield timestamp, diff, self._get_quotes_from_kline_open( self._data["quotes"][symbol_list[0]], timestamp, item) # K线刚生成时的数据都为开盘价 timestamp = item["datetime"] + dur - 1000 \ if dur < 86400000000000 else _get_trading_day_start_time(item["datetime"] + dur) - 1000 if timestamp > self._end_dt: # 超过结束时间 return diff = { "klines": { symbol_list[0]: { str(dur): { "data": { str(current_id): item, } } } } } for i, symbol in enumerate(symbol_list): if i == 0: continue other_id = binding.get(symbol, {}).get(str(current_id), -1) if other_id >= 0: diff["klines"][symbol] = { str(dur): { "data": { str(other_id): {k: v for k, v in serials[i]["data"].get(str(other_id), {}).items()} } } } yield timestamp, diff, self._get_quotes_from_kline(self._data["quotes"][symbol_list[0]], timestamp, item) # K线结束时生成quote数据 current_id += 1 finally: # 释放chart资源 chart_info["ins_list"] = "" await self._md_send_chan.send(chart_info.copy()) chart_info["chart_id"] = chart_id_b if chart_info["chart_id"] == chart_id_a else chart_id_a await self._md_send_chan.send(chart_info.copy())
def __init__(self, api: TqApi, symbol_list: Union[str, List[str]], dur_sec: int, start_dt: Union[date, datetime], end_dt: Union[date, datetime], csv_file_name: str, adj_type: Union[str, None] = None) -> None: """ 创建历史数据下载器实例 Args: api (TqApi): TqApi实例,该下载器将使用指定的api下载数据 symbol_list (str/list of str): 需要下载数据的合约代码,当指定多个合约代码时将其他合约按第一个合约的交易时间对齐 dur_sec (int): 数据周期,以秒为单位。例如: 1分钟线为60,1小时线为3600,日线为86400,Tick数据为0 start_dt (date/datetime): 起始时间, 如果类型为 date 则指的是交易日, 如果为 datetime 则指的是具体时间点 end_dt (date/datetime): 结束时间, 如果类型为 date 则指的是交易日, 如果为 datetime 则指的是具体时间点 csv_file_name (str): 输出 csv 的文件名 adj_type (str/None): 复权计算方式,默认值为 None。"F" 为前复权;"B" 为后复权;None 表示不复权。只对股票、基金合约有效。 Example:: from datetime import datetime, date from contextlib import closing from tqsdk import TqApi, TqAuth, TqSim from tqsdk.tools import DataDownloader api = TqApi(auth=TqAuth("信易账户", "账户密码")) download_tasks = {} # 下载从 2018-01-01 到 2018-09-01 的 SR901 日线数据 download_tasks["SR_daily"] = DataDownloader(api, symbol_list="CZCE.SR901", dur_sec=24*60*60, start_dt=date(2018, 1, 1), end_dt=date(2018, 9, 1), csv_file_name="SR901_daily.csv") # 下载从 2017-01-01 到 2018-09-01 的 rb主连 5分钟线数据 download_tasks["rb_5min"] = DataDownloader(api, symbol_list="*****@*****.**", dur_sec=5*60, start_dt=date(2017, 1, 1), end_dt=date(2018, 9, 1), csv_file_name="rb_5min.csv") # 下载从 2018-01-01凌晨6点 到 2018-06-01下午4点 的 cu1805,cu1807,IC1803 分钟线数据,所有数据按 cu1805 的时间对齐 # 例如 cu1805 夜盘交易时段, IC1803 的各项数据为 N/A # 例如 cu1805 13:00-13:30 不交易, 因此 IC1803 在 13:00-13:30 之间的K线数据会被跳过 download_tasks["cu_min"] = DataDownloader(api, symbol_list=["SHFE.cu1805", "SHFE.cu1807", "CFFEX.IC1803"], dur_sec=60, start_dt=datetime(2018, 1, 1, 6, 0 ,0), end_dt=datetime(2018, 6, 1, 16, 0, 0), csv_file_name="cu_min.csv") # 下载从 2018-05-01凌晨0点 到 2018-06-01凌晨0点 的 T1809 盘口Tick数据 download_tasks["T_tick"] = DataDownloader(api, symbol_list=["CFFEX.T1809"], dur_sec=0, start_dt=datetime(2018, 5, 1), end_dt=datetime(2018, 6, 1), csv_file_name="T1809_tick.csv") # 使用with closing机制确保下载完成后释放对应的资源 with closing(api): while not all([v.is_finished() for v in download_tasks.values()]): api.wait_update() print("progress: ", { k:("%.2f%%" % v.get_progress()) for k,v in download_tasks.items() }) """ self._api = api if not self._api._auth._has_feature("tq_dl"): raise Exception( "您的账户不支持下载历史数据功能,需要购买专业版本后使用。升级网址:https://account.shinnytech.com" ) if isinstance(start_dt, datetime): self._start_dt_nano = int(start_dt.timestamp() * 1e9) else: self._start_dt_nano = _get_trading_day_start_time( int( datetime(start_dt.year, start_dt.month, start_dt.day).timestamp()) * 1000000000) if isinstance(end_dt, datetime): self._end_dt_nano = int(end_dt.timestamp() * 1e9) else: self._end_dt_nano = _get_trading_day_end_time( int( datetime(end_dt.year, end_dt.month, end_dt.day).timestamp()) * 1000000000) self._current_dt_nano = self._start_dt_nano self._symbol_list = symbol_list if isinstance(symbol_list, list) else [symbol_list] # 下载合约超时时间(默认 30s),已下市的没有交易的合约,超时时间可以设置短一点(2s),用户不希望自己的程序因为没有下载到数据而中断 self._timeout_seconds = 2 if any( [symbol in DEAD_INS for symbol in self._symbol_list]) else 30 self._dur_nano = dur_sec * 1000000000 if self._dur_nano == 0 and len(self._symbol_list) != 1: raise Exception("Tick序列不支持多合约") if adj_type not in [None, "F", "B", "FORWARD", "BACK"]: raise Exception( "adj_type 参数只支持 None (不复权) | 'F' (前复权) | 'B' (后复权)") self._adj_type = adj_type[0] if adj_type else adj_type self._csv_file_name = csv_file_name self._csv_header = self._get_headers() self._dividend_cache = {} # 缓存合约对应的复权系数矩阵,每个合约只计算一次 self._data_series = None self._task = self._api.create_task(self._run())