def get_history_bar(gateway, symbol, exchange: "Exchange"): """ 获取历史数据并存入数据库 :param gateway: :param symbol: :param exchange: :return: """ gevent.sleep(1) # 1min history = HistoryRequest(symbol=symbol, exchange=exchange, start=None, interval=Interval.MINUTE) gateway.query_history(history) gevent.sleep(2) # 1hour history = HistoryRequest(symbol=symbol, exchange=exchange, start=None, interval=Interval.HOUR) gateway.query_history(history) gevent.sleep(2) # 1day history = HistoryRequest(symbol=symbol, exchange=exchange, start=None, interval=Interval.DAILY) gateway.query_history(history) gevent.sleep(2)
def _query_history( self, vt_symbol: str, interval: Interval, start: datetime, end: datetime ) -> None: """""" contract: ContractData = self.main_engine.get_contract(vt_symbol) req = HistoryRequest( symbol=contract.symbol, exchange=contract.exchange, interval=interval, start=start, end=end ) if contract.history_data: data = self.main_engine.query_history(req, contract.gateway_name) else: data = rqdata_client.query_history(req) event = Event(EVENT_CHART_HISTORY, data) self.event_engine.put(event)
def download_bar_data(self, symbol: str, exchange: Exchange, interval: str, start: datetime) -> int: """ Query bar data from RQData. """ req = HistoryRequest(symbol=symbol, exchange=exchange, interval=Interval(interval), start=start, end=datetime.now()) vt_symbol = f"{symbol}.{exchange.value}" contract = self.main_engine.get_contract(vt_symbol) # If history data provided in gateway, then query if contract and contract.history_data: data = self.main_engine.query_history(req, contract.gateway_name) # Otherwise use RQData to query data else: if not rqdata_client.inited: rqdata_client.init() data = rqdata_client.query_history(req) if data: database_manager.save_bar_data(data) return (len(data)) return 0
def run_single_backtesting(args): engine = BacktestingEngine() req = HistoryRequest(symbol=args.symbol, exchange=Exchange(args.exchange), interval=Interval(args.interval), start=datetime.strptime(args.startdate, '%Y-%m-%d'), end=datetime.strptime(args.enddate, '%Y-%m-%d')) setting = txt_to_dic(args.backtesting_setting_file) strategy_class = STRATEGIES[args.strategy_class] engine.set_parameters(vt_symbol=req.vt_symbol, interval=req.interval, start=req.start, end=req.end, rate=args.rate, slippage=args.slippage, size=args.size, pricetick=args.pricetick, capital=args.capital) engine.add_strategy(strategy_class, setting) engine.load_data() engine.run_backtesting() df = engine.calculate_result() engine.calculate_statistics(df) engine.show_chart(df)
def load_bar(self, vt_symbol: str, days: int, interval: Interval, callback: Callable[[BarData], None]): """""" symbol, exchange = extract_vt_symbol(vt_symbol) end = datetime.now() start = end - timedelta(days) # Query bars from gateway if available contract = self.main_engine.get_contract(vt_symbol) if contract and contract.history_data: req = HistoryRequest(symbol=symbol, exchange=exchange, interval=interval, start=start, end=end) bars = self.main_engine.query_history(req, contract.gateway_name) # Try to query bars from RQData, if not found, load from database. else: bars = self.query_bar_from_rq(symbol, exchange, interval, start, end) if not bars: bars = database_manager.load_bar_data( symbol=symbol, exchange=exchange, interval=interval, start=start, end=end, ) for bar in bars: callback(bar)
def download_bar_data( self, symbol: str, exchange: Exchange, interval: str, start: datetime ) -> int: """ """ req = HistoryRequest( symbol=symbol, exchange=exchange, interval=Interval(interval), start=start, end=datetime.now() ) vt_symbol = f"{symbol}.{exchange.value}" contract = self.main_engine.get_contract(vt_symbol) # If history data provided in gateway, then query if contract and contract.history_data: data = self.main_engine.query_history( req, contract.gateway_name ) if data: database_manager.save_bar_data(data) return(len(data)) return 0
def download_all(self): """ 使用tushare下载A股股票全市场日线数据 :return: """ log.info("开始下载A股股票全市场日线数据") if self.symbols is not None: with tqdm(total=len(self.symbols)) as pbar: for tscode, list_date in zip(self.symbols['ts_code'], self.symbols['list_date']): symbol, exchange = to_split_ts_codes(tscode) pbar.set_description_str("下载A股日线数据股票代码:" + tscode) start_date = datetime.strptime(list_date, TS_DATE_FORMATE) req = HistoryRequest(symbol=symbol, exchange=exchange, start=start_date, end=datetime.now(), interval=Interval.DAILY) bardata = self.tushare_client.query_history(req=req) if bardata: try: database_manager.save_bar_data(bardata) except Exception as ex: log.error(tscode + "数据存入数据库异常") log.error(ex) traceback.print_exc() pbar.update(1) log.info(pbar.desc) log.info("A股股票全市场日线数据下载完毕")
def run_downloading(self, vt_symbol: str, interval: str, start: datetime, end: datetime): """ Query bar data from RQData. """ self.write_log(f"{vt_symbol}-{interval}开始下载历史数据") symbol, exchange = extract_vt_symbol(vt_symbol) req = HistoryRequest(symbol=symbol, exchange=exchange, interval=Interval(interval), start=start, end=end) contract = self.main_engine.get_contract(vt_symbol) # If history data provided in gateway, then query if contract and contract.history_data: data = self.main_engine.query_history(req, contract.gateway_name) # Otherwise use RQData to query data else: data = rqdata_client.query_history(req) if data: database_manager.save_bar_data(data) self.write_log(f"{vt_symbol}-{interval}历史数据下载完成") else: self.write_log(f"数据下载失败,无法获取{vt_symbol}的历史数据") # Clear thread object handler. self.thread = None
def load_bar(self, vt_symbol: str, days: int, interval: Interval) -> List[BarData]: """""" symbol, exchange = extract_vt_symbol(vt_symbol) end = datetime.now(get_localzone()) start = end - timedelta(days) contract: ContractData = self.main_engine.get_contract(vt_symbol) data = [] # Query bars from gateway if available if contract and contract.history_data: req = HistoryRequest(symbol=symbol, exchange=exchange, interval=interval, start=start, end=end) data = self.main_engine.query_history(req, contract.gateway_name) # Try to query bars from RQData, if not found, load from database. else: data = self.query_bar_from_rq(symbol, exchange, interval, start, end) if not data: data = database_manager.load_bar_data( symbol=symbol, exchange=exchange, interval=interval, start=start, end=end, ) return data
def update_all(self, symbol, exchange, trade_datas, start, end=None): self._symbol = symbol self._exchange = exchange self._start = start interval = self._interval tp = self.interval2timdelta(self._interval) backward_n = max(60 * tp, dt.timedelta(hours=25)) end = start + backward_n if end is None else end history_data = database_manager.load_bar_data(symbol, exchange, interval, start=start, end=end) self.trade_datas = trade_datas if len(history_data) > 0 and len(history_data) / ( (end - start).total_seconds() / 60) > 0.7: self.chart.update_all(history_data, trade_datas, []) else: req = HistoryRequest(symbol, exchange, start, end, interval) gateway = self.main_engine.get_gateway('IB') if gateway and gateway.api.status: self.history_data = history_data = gateway.query_history(req) self.chart.update_all(history_data, trade_datas, []) database_manager.save_bar_data(history_data) if len(getattr(self, 'history_data', [])) > 0: self._end = self.history_data[-1].datetime
def rq_download( self, vt_symbol: str, interval: str, start: datetime, end: datetime, ): rqdata_client.init() symbol, exchange = extract_vt_symbol(vt_symbol) req = HistoryRequest(symbol=symbol, exchange=exchange, interval=Interval(interval), start=start, end=end) # print(req) data = rqdata_client.query_history(req) if data: database_manager.save_bar_data(data) print(f"{vt_symbol}-{interval} 历史数据下载完成") else: print(f"数据下载失败,无法得到 {vt_symbol} 的数据")
def download_tick_data( self, symbol: str, exchange: Exchange, start: datetime ) -> int: """ Query tick data from RQData. """ req = HistoryRequest( symbol=symbol, exchange=exchange, start=start, end=datetime.now() ) if not rqdata_client.inited: rqdata_client.init() data = rqdata_client.query_history(req) if data: database_manager.save_tick_data(data) return(len(data)) return 0
def check_update_all(self): """ 这个方法太慢了,不建议调用。 这个方法用于本地数据库已经建立,但可能有部分数据缺失时使用 使用tushare检查更新所有的A股股票全市场日线数据 检查哪一个交易日的数据是缺失的,补全它 检查上市后是否每个交易日都有数据,若存在某一交易日无数据,尝试从tushare查询该日数据,若仍无,则说明当天停盘 :return: """ log.info("开始检查更新所有的A股股票全市场日线数据") if self.symbols is not None: with tqdm(total=len(self.symbols)) as pbar: for tscode, list_date in zip(self.symbols['ts_code'], self.symbols['list_date']): pbar.set_description_str("正在检查A股日线数据,股票代码:" + tscode) symbol, exchange = to_split_ts_codes(tscode) local_bar = database_manager.load_bar_data( symbol=symbol, exchange=exchange, interval=Interval.DAILY, start=datetime.strptime(list_date, TS_DATE_FORMATE), end=datetime.now()) local_bar_dates = [ bar.datetime.strftime(TS_DATE_FORMATE) for bar in local_bar ] index = (self.trade_cal[exchange.value][( self.trade_cal[exchange.value].cal_date == list_date)]) trade_cal = self.trade_cal[ exchange.value].iloc[index.index[0]:] for trade_date in trade_cal['cal_date']: if trade_date not in local_bar_dates: req = HistoryRequest(symbol=symbol, exchange=exchange, start=datetime.strptime( trade_date, TS_DATE_FORMATE), end=datetime.strptime( trade_date, TS_DATE_FORMATE), interval=Interval.DAILY) bardata = self.tushare_client.query_history( req=req) if bardata: log.info(tscode + "本地数据库缺失:" + trade_date) try: database_manager.save_bar_data(bardata) except Exception as ex: log.error(tscode + "数据存入数据库异常") log.error(ex) traceback.print_exc() pbar.update(1) log.info(pbar.desc) log.info("A股股票全市场日线数据检查更新完毕")
def load_market_trade(self, vt_symbol: str, callback: Callable): contract = self.main_engine.get_contract(vt_symbol) symbol, exchange = extract_vt_symbol(vt_symbol) req = HistoryRequest(symbol=symbol, exchange=exchange, start=datetime.now()) trades = self.main_engine.query_market_trade(req=req, gateway_name=contract.gateway_name) for trade in trades: callback(trade)
def test1(): req = HistoryRequest(exchange=Exchange.SMART, symbol="goog", interval=Interval.MINUTE, start=datetime.datetime(2019, 7, 9, 9), end=datetime.datetime(2019, 8, 16, 4)) bar_data = rq.rqdata_client.query_history(req) print(bar_data)
def data_record(start, end, vt_symbol): from vnpy.trader.database import database_manager from vnpy.gateway.ib.ib_gateway import IbGateway from vnpy.trader.utility import load_json from vnpy.trader.object import HistoryRequest from vnpy.trader.constant import Interval, Exchange from dateutil import parser from vnpy.event.engine import EventEngine from vnpy.trader.event import EVENT_LOG vt_symbol = vt_symbol symbol, exchange = vt_symbol.split('.') if not start and not end: offset = 0 if dt.datetime.now().time() > dt.time(17, 0) else 1 start = (dt.datetime.today() - dt.timedelta(days=offset + 1)).replace( hour=17, minute=0, second=0, microsecond=0) end = (dt.datetime.today() - dt.timedelta(days=offset)).replace( hour=17, minute=0, second=0, microsecond=0) else: start = parser.parse(start) end = parser.parse(end) if end else end ib_settings = load_json('connect_ib.json') ib_settings["客户号"] += 4 recorder_engine = EventEngine() def log(event): data = event.data print(data.level, data.msg) recorder_engine.register(EVENT_LOG, log) ib = IbGateway(recorder_engine) try: recorder_engine.start() ib.connect(ib_settings) if ib.api.client.isConnected(): req = HistoryRequest(symbol, Exchange(exchange), start, end, Interval.MINUTE) ib.write_log(f'发起请求#{vt_symbol}, {start}至{end}') his_data = ib.query_history(req) ib.write_log( f'获得数据#{vt_symbol}, {his_data[0].datetime}至{his_data[-1].datetime}, 共{len(his_data)}条' ) database_manager.save_bar_data(his_data) ib.write_log(f'成功入库') else: ib.write_log('连接失败!请检查客户号是否被占用或IP是否正确') except Exception as e: raise e finally: ib.close() recorder_engine.stop()
def load_history_data(self, req: HistoryRequest): """""" self.output("开始加载历史数据") if not req.end: req.end = datetime.now() if req.start >= req.end: self.output("起始日期必须小于结束日期") return self.history_data.clear() # Clear previously loaded history data # Load 30 days of data each time and allow for progress update total_days = (req.end - req.start).days progress_days = max(int(total_days / 10), 1) progress_delta = timedelta(days=progress_days) interval_delta = INTERVAL_DELTA_MAP[req.interval] start = req.start end = req.start + progress_delta progress = 0 while start < req.end: progress_bar = "#" * int(progress * 10 + 1) self.output(f"加载进度:{progress_bar} [{progress:.0%}]") end = min(end, req.end) # Make sure end time stays within set range if self.mode == BacktestingMode.BAR: data = load_bar_data( req.symbol, req.exchange, req.interval, start, end ) else: data = load_tick_data( req.symbol, req.exchange, start, end ) self.history_data.extend(data) progress += progress_days / total_days progress = min(progress, 1) start = end + interval_delta end += progress_delta self.output(f"{req.start}至{req.end}历史数据加载完成,数据量:{len(self.history_data)}")
def diff(): req = HistoryRequest( symbol=symbol, exchange=exchange, interval=Interval.MINUTE, start=start_date, end=end_date ) data = rqdata_client.query_history(req) print(data)
def write(): req = HistoryRequest( symbol=symbol, exchange=exchange, interval=Interval.MINUTE, start=start_date, end=end_date ) data = rqdata_client.query_history(req) database_manager.save_bar_data(data)
def main(): """""" qapp = create_qapp() event_engine = EventEngine() main_engine = MainEngine(event_engine) main_engine.add_gateway(BinancesGateway) engine_name = 'BINANCES' setting = { "key": "bEZCKVAkJYugYErdlg5GAj7PGgh72ieFIVtDVYgUwJuE7M0vLFM7lN7JuQPYB0AL", "secret": "LGZXmPUe4iXqOWlmjiUHGvf7XsdLSBWe5HfXauJoxu8qR9RTeZMUqzTOhcRiAK9K", "会话数": 3, "服务器": "REAL", "合约模式": "正向", "代理地址": "", "代理端口": 0 } main_engine.connect(setting, engine_name) yearlists = {'2020': '2021'} # all existed currency futures symbols = SYMBOLS # symbol = symbols[0] # quote = SubscribeRequest(symbol, BinanceGateway.exchanges[0]) # df = main_engine.subscribe(quote,'BINANCE') INTERV = 'MINUTE' # INTERV = 'YEAR' for symbol in symbols: this_directory = 'C:\\Workspace\\crypto\\data\\' + symbol if not os.path.exists(this_directory): os.makedirs(this_directory) for key, var in yearlists.items(): startdate = pd.to_datetime(key + '-01-01') enddate = pd.to_datetime(var + '-01-06') print('query {} minute history '.format(symbol)) quote = HistoryRequest(symbol, BinancesGateway.exchanges[0], start=startdate, end=enddate, interval=Interval.MINUTE) df = main_engine.query_history(quote, engine_name) # HOUR this_model_file = this_directory + '\\' + symbol + '_FUTURES' + '_' + INTERV + '_' + key + '.pkl' # MINUTE # this_model_file = this_directory + '\\' + symbol + '_' + INTERV + '_all' + '.pkl' with open(this_model_file, 'wb') as f: # first save model pickle.dump(df, f)
def query_bar_from_rq(self, symbol: str, exchange: Exchange, interval: Interval, start: datetime, end: datetime): """ Query bar data from RQData. """ req = HistoryRequest(symbol=symbol, exchange=exchange, interval=interval, start=start, end=end) data = rqdata_client.query_history(req) return data
def test3(): event_engine = EventEngine() gate = HbdmGateway(event_engine) setting = { "API Key": "mjlpdje3ld-701b9727-9ae52956-3ef1e", "Secret Key": "b4df4824-124ec192-55786af1-a397a", "会话数": 3, "代理地址": "127.0.0.1", "代理端口": "1080", } gate.connect(setting) time.sleep(3) print(symbol_type_map) if len(symbol_type_map) <= 0: return req = HistoryRequest(exchange=Exchange.HUOBI, symbol="BCH190830", interval=Interval.MINUTE, start=datetime.datetime(2019, 8, 1, 0), end=datetime.datetime(2019, 8, 17, 16)) bar_data = gate.query_history(req) tb = TestBar() date = [] for v in bar_data: tb.am.update_bar(v) date.append(v.datetime) dt = date close = tb.am.close_array rect1 = [0.14, 0.35, 0.77, 0.6] # [左, 下, 宽, 高] 规定的矩形区域 (全部是0~1之间的数,表示比例) rect2 = [0.14, 0.05, 0.77, 0.2] fig = plt.figure() ax1 = plt.axes(rect1) ax2 = plt.axes(rect2) # ax1 = fig.add_subplot(2,1,1) print(dt) print(close) ax1.plot(dt, close) v_list, p_list = tb.am.wave() for i in range(len(p_list)): p = p_list[i] ax1.plot([dt[p]], [close[p]], 'o') ax1.annotate(close[p], xy=(dt[p], close[p])) print(close) sma120 = talib.SMA(close, timeperiod=120) ax1.plot(dt, sma120) # ax2 = fig.add_subplot(2,1,2) kdj_data = tb.am.kdj() ax2.plot(dt, kdj_data["k"], "r") ax2.plot(dt, kdj_data["d"], "b")
def query_bar_from_datafeed(self, symbol: str, exchange: Exchange, interval: Interval, start: datetime, end: datetime): """ Query bar data from datafeed. """ req = HistoryRequest(symbol=symbol, exchange=exchange, interval=interval, start=start, end=end) data = self.datafeed.query_bar_history(req) return data
def query_bar_from_tq( self, symbol: str, exchange: Exchange, interval: Interval, start: datetime, end: datetime ): req = HistoryRequest( symbol=symbol, exchange=exchange, interval=interval, start=start, end=end ) tqdata_api = TqdataApi() data = tqdata_api.query_history(req) return data
def get_historical_data(self, contract, end, bar_count, interval): total_minutes = { Interval.MINUTE: 1, Interval.HOUR: 60 }[interval] * bar_count start = (end if end else dt.datetime.now( get_localzone())) - dt.timedelta(minutes=total_minutes) req = HistoryRequest(contract.symbol, contract.exchange, start=start, end=end, interval=interval) his_data = self.main_engine.query_history(req, contract.gateway_name) return his_data
def download_history_data(symbol, exchange): print(symbol, exchange) begin_date = datetime.strptime("2019-09-19", "%Y-%m-%d") for i in range(1): start_date = begin_date.replace(year=begin_date.year + i) # end_date = begin_date.replace(year=begin_date.year + i + 1) end_date = datetime.now() print(start_date, end_date) req = HistoryRequest(symbol=symbol, exchange=Exchange(exchange), interval=Interval("1m"), start=start_date, end=end_date) data = mddata_client.query_history(req)
def downloading_history_data(args): ''' download stocks or futures bar data. ''' req = HistoryRequest(symbol=args.symbol, exchange=Exchange(args.exchange), interval=Interval(args.interval), start=datetime.strptime(args.startdate, '%Y-%m-%d'), end=datetime.strptime(args.enddate, '%Y-%m-%d')) event_engine = EventEngine() main_engine = MainEngine(event_engine) main_engine.add_gateway(CtpGateway) cta_backtester_engine = main_engine.add_app(CtaBacktesterApp) cta_backtester_engine.start_downloading(req.vt_symbol, req.interval, req.start, req.end)
def run_downloading(self, vt_symbol: str, interval: str, start: datetime, end: datetime): """ Query bar data from RQData. """ self.write_log(f"{vt_symbol}-{interval}开始下载历史数据") try: symbol, exchange = extract_vt_symbol(vt_symbol) except ValueError: self.write_log(f"{vt_symbol}解析失败,请检查交易所后缀") self.thread = None return req = HistoryRequest(symbol=symbol, exchange=exchange, interval=Interval(interval), start=start, end=end) contract = self.main_engine.get_contract(vt_symbol) try: # If history data provided in gateway, then query if contract and contract.history_data: data = self.main_engine.query_history(req, contract.gateway_name) # Otherwise use RQData to query data else: if SETTINGS["rqdata.username"]: data = rqdata_client.query_history(req) elif SETTINGS["tqdata.username"]: data = tqdata_client.query_history(req) else: data = [] if data: database_manager.save_bar_data(data) self.write_log(f"{vt_symbol}-{interval}历史数据下载完成") else: self.write_log(f"数据下载失败,无法获取{vt_symbol}的历史数据") except Exception: msg = f"数据下载失败,触发异常:\n{traceback.format_exc()}" self.write_log(msg) # Clear thread object handler. self.thread = None
def get_jq_bars(self, symbol: str, interval: str, count: int): """""" contract = self.main_engine.get_current_contract() if not contract: return [] req = HistoryRequest( symbol=contract.symbol, count=count, exchange=contract.exchange, interval=Interval(interval), start="", end="", ) bars_list, bars_df = jqdata_client.query_history(req) return bars_list
def query_bar_from_rq(self, symbol: str, exchange: Exchange, interval: Interval, start: datetime, end: datetime): """ Query bar data from RQData. """ req = HistoryRequest(symbol=symbol, exchange=exchange, interval=interval, start=start, end=end) # data = mddata_client.query_history(req) try: data = mddata_client.query_history(req) except Exception as ex: self.write_log(f"{symbol}.{exchange.value}合约下载失败:{ex.args}") return None return data