def get_history_bar(gateway, symbol, exchange: "Exchange"):
"""
获取历史数据并存入数据库
:param gateway:
:param symbol:
:param exchange:
:return:
"""
gevent.sleep(1)
# 1min
history = HistoryRequest(symbol=symbol,
exchange=exchange,
start=None,
interval=Interval.MINUTE)
gateway.query_history(history)
gevent.sleep(2)
# 1hour
history = HistoryRequest(symbol=symbol,
exchange=exchange,
start=None,
interval=Interval.HOUR)
gateway.query_history(history)
gevent.sleep(2)
# 1day
history = HistoryRequest(symbol=symbol,
exchange=exchange,
start=None,
interval=Interval.DAILY)
gateway.query_history(history)
gevent.sleep(2)
def _query_history(
self,
vt_symbol: str,
interval: Interval,
start: datetime,
end: datetime
) -> None:
""""""
contract: ContractData = self.main_engine.get_contract(vt_symbol)
req = HistoryRequest(
symbol=contract.symbol,
exchange=contract.exchange,
interval=interval,
start=start,
end=end
)
if contract.history_data:
data = self.main_engine.query_history(req, contract.gateway_name)
else:
data = rqdata_client.query_history(req)
event = Event(EVENT_CHART_HISTORY, data)
self.event_engine.put(event)
def download_bar_data(self, symbol: str, exchange: Exchange, interval: str,
start: datetime) -> int:
"""
Query bar data from RQData.
"""
req = HistoryRequest(symbol=symbol,
exchange=exchange,
interval=Interval(interval),
start=start,
end=datetime.now())
vt_symbol = f"{symbol}.{exchange.value}"
contract = self.main_engine.get_contract(vt_symbol)
# If history data provided in gateway, then query
if contract and contract.history_data:
data = self.main_engine.query_history(req, contract.gateway_name)
# Otherwise use RQData to query data
else:
if not rqdata_client.inited:
rqdata_client.init()
data = rqdata_client.query_history(req)
if data:
database_manager.save_bar_data(data)
return (len(data))
return 0
def run_single_backtesting(args):
engine = BacktestingEngine()
req = HistoryRequest(symbol=args.symbol,
exchange=Exchange(args.exchange),
interval=Interval(args.interval),
start=datetime.strptime(args.startdate, '%Y-%m-%d'),
end=datetime.strptime(args.enddate, '%Y-%m-%d'))
setting = txt_to_dic(args.backtesting_setting_file)
strategy_class = STRATEGIES[args.strategy_class]
engine.set_parameters(vt_symbol=req.vt_symbol,
interval=req.interval,
start=req.start,
end=req.end,
rate=args.rate,
slippage=args.slippage,
size=args.size,
pricetick=args.pricetick,
capital=args.capital)
engine.add_strategy(strategy_class, setting)
engine.load_data()
engine.run_backtesting()
df = engine.calculate_result()
engine.calculate_statistics(df)
engine.show_chart(df)
def load_bar(self, vt_symbol: str, days: int, interval: Interval,
callback: Callable[[BarData], None]):
""""""
symbol, exchange = extract_vt_symbol(vt_symbol)
end = datetime.now()
start = end - timedelta(days)
# Query bars from gateway if available
contract = self.main_engine.get_contract(vt_symbol)
if contract and contract.history_data:
req = HistoryRequest(symbol=symbol,
exchange=exchange,
interval=interval,
start=start,
end=end)
bars = self.main_engine.query_history(req, contract.gateway_name)
# Try to query bars from RQData, if not found, load from database.
else:
bars = self.query_bar_from_rq(symbol, exchange, interval, start,
end)
if not bars:
bars = database_manager.load_bar_data(
symbol=symbol,
exchange=exchange,
interval=interval,
start=start,
end=end,
)
for bar in bars:
callback(bar)
def download_bar_data(
self,
symbol: str,
exchange: Exchange,
interval: str,
start: datetime
) -> int:
"""
"""
req = HistoryRequest(
symbol=symbol,
exchange=exchange,
interval=Interval(interval),
start=start,
end=datetime.now()
)
vt_symbol = f"{symbol}.{exchange.value}"
contract = self.main_engine.get_contract(vt_symbol)
# If history data provided in gateway, then query
if contract and contract.history_data:
data = self.main_engine.query_history(
req, contract.gateway_name
)
if data:
database_manager.save_bar_data(data)
return(len(data))
return 0
def download_all(self):
"""
使用tushare下载A股股票全市场日线数据
:return:
"""
log.info("开始下载A股股票全市场日线数据")
if self.symbols is not None:
with tqdm(total=len(self.symbols)) as pbar:
for tscode, list_date in zip(self.symbols['ts_code'],
self.symbols['list_date']):
symbol, exchange = to_split_ts_codes(tscode)
pbar.set_description_str("下载A股日线数据股票代码:" + tscode)
start_date = datetime.strptime(list_date, TS_DATE_FORMATE)
req = HistoryRequest(symbol=symbol,
exchange=exchange,
start=start_date,
end=datetime.now(),
interval=Interval.DAILY)
bardata = self.tushare_client.query_history(req=req)
if bardata:
try:
database_manager.save_bar_data(bardata)
except Exception as ex:
log.error(tscode + "数据存入数据库异常")
log.error(ex)
traceback.print_exc()
pbar.update(1)
log.info(pbar.desc)
log.info("A股股票全市场日线数据下载完毕")
def run_downloading(self, vt_symbol: str, interval: str, start: datetime,
end: datetime):
"""
Query bar data from RQData.
"""
self.write_log(f"{vt_symbol}-{interval}开始下载历史数据")
symbol, exchange = extract_vt_symbol(vt_symbol)
req = HistoryRequest(symbol=symbol,
exchange=exchange,
interval=Interval(interval),
start=start,
end=end)
contract = self.main_engine.get_contract(vt_symbol)
# If history data provided in gateway, then query
if contract and contract.history_data:
data = self.main_engine.query_history(req, contract.gateway_name)
# Otherwise use RQData to query data
else:
data = rqdata_client.query_history(req)
if data:
database_manager.save_bar_data(data)
self.write_log(f"{vt_symbol}-{interval}历史数据下载完成")
else:
self.write_log(f"数据下载失败，无法获取{vt_symbol}的历史数据")
# Clear thread object handler.
self.thread = None
def load_bar(self, vt_symbol: str, days: int,
interval: Interval) -> List[BarData]:
""""""
symbol, exchange = extract_vt_symbol(vt_symbol)
end = datetime.now(get_localzone())
start = end - timedelta(days)
contract: ContractData = self.main_engine.get_contract(vt_symbol)
data = []
# Query bars from gateway if available
if contract and contract.history_data:
req = HistoryRequest(symbol=symbol,
exchange=exchange,
interval=interval,
start=start,
end=end)
data = self.main_engine.query_history(req, contract.gateway_name)
# Try to query bars from RQData, if not found, load from database.
else:
data = self.query_bar_from_rq(symbol, exchange, interval, start,
end)
if not data:
data = database_manager.load_bar_data(
symbol=symbol,
exchange=exchange,
interval=interval,
start=start,
end=end,
)
return data
def update_all(self, symbol, exchange, trade_datas, start, end=None):
self._symbol = symbol
self._exchange = exchange
self._start = start
interval = self._interval
tp = self.interval2timdelta(self._interval)
backward_n = max(60 * tp, dt.timedelta(hours=25))
end = start + backward_n if end is None else end
history_data = database_manager.load_bar_data(symbol,
exchange,
interval,
start=start,
end=end)
self.trade_datas = trade_datas
if len(history_data) > 0 and len(history_data) / (
(end - start).total_seconds() / 60) > 0.7:
self.chart.update_all(history_data, trade_datas, [])
else:
req = HistoryRequest(symbol, exchange, start, end, interval)
gateway = self.main_engine.get_gateway('IB')
if gateway and gateway.api.status:
self.history_data = history_data = gateway.query_history(req)
self.chart.update_all(history_data, trade_datas, [])
database_manager.save_bar_data(history_data)
if len(getattr(self, 'history_data', [])) > 0:
self._end = self.history_data[-1].datetime
def rq_download(
self,
vt_symbol: str,
interval: str,
start: datetime,
end: datetime,
):
rqdata_client.init()
symbol, exchange = extract_vt_symbol(vt_symbol)
req = HistoryRequest(symbol=symbol,
exchange=exchange,
interval=Interval(interval),
start=start,
end=end)
# print(req)
data = rqdata_client.query_history(req)
if data:
database_manager.save_bar_data(data)
print(f"{vt_symbol}-{interval} 历史数据下载完成")
else:
print(f"数据下载失败，无法得到 {vt_symbol} 的数据")
def download_tick_data(
self,
symbol: str,
exchange: Exchange,
start: datetime
) -> int:
"""
Query tick data from RQData.
"""
req = HistoryRequest(
symbol=symbol,
exchange=exchange,
start=start,
end=datetime.now()
)
if not rqdata_client.inited:
rqdata_client.init()
data = rqdata_client.query_history(req)
if data:
database_manager.save_tick_data(data)
return(len(data))
return 0
def check_update_all(self):
"""
这个方法太慢了，不建议调用。
这个方法用于本地数据库已经建立，但可能有部分数据缺失时使用
使用tushare检查更新所有的A股股票全市场日线数据
检查哪一个交易日的数据是缺失的，补全它
检查上市后是否每个交易日都有数据，若存在某一交易日无数据，尝试从tushare查询该日数据，若仍无，则说明当天停盘
:return:
"""
log.info("开始检查更新所有的A股股票全市场日线数据")
if self.symbols is not None:
with tqdm(total=len(self.symbols)) as pbar:
for tscode, list_date in zip(self.symbols['ts_code'],
self.symbols['list_date']):
pbar.set_description_str("正在检查A股日线数据，股票代码:" + tscode)
symbol, exchange = to_split_ts_codes(tscode)
local_bar = database_manager.load_bar_data(
symbol=symbol,
exchange=exchange,
interval=Interval.DAILY,
start=datetime.strptime(list_date, TS_DATE_FORMATE),
end=datetime.now())
local_bar_dates = [
bar.datetime.strftime(TS_DATE_FORMATE)
for bar in local_bar
]
index = (self.trade_cal[exchange.value][(
self.trade_cal[exchange.value].cal_date == list_date)])
trade_cal = self.trade_cal[
exchange.value].iloc[index.index[0]:]
for trade_date in trade_cal['cal_date']:
if trade_date not in local_bar_dates:
req = HistoryRequest(symbol=symbol,
exchange=exchange,
start=datetime.strptime(
trade_date,
TS_DATE_FORMATE),
end=datetime.strptime(
trade_date,
TS_DATE_FORMATE),
interval=Interval.DAILY)
bardata = self.tushare_client.query_history(
req=req)
if bardata:
log.info(tscode + "本地数据库缺失：" + trade_date)
try:
database_manager.save_bar_data(bardata)
except Exception as ex:
log.error(tscode + "数据存入数据库异常")
log.error(ex)
traceback.print_exc()
pbar.update(1)
log.info(pbar.desc)
log.info("A股股票全市场日线数据检查更新完毕")
def load_market_trade(self, vt_symbol: str, callback: Callable):
contract = self.main_engine.get_contract(vt_symbol)
symbol, exchange = extract_vt_symbol(vt_symbol)
req = HistoryRequest(symbol=symbol, exchange=exchange, start=datetime.now())
trades = self.main_engine.query_market_trade(req=req, gateway_name=contract.gateway_name)
for trade in trades:
callback(trade)
def test1():
req = HistoryRequest(exchange=Exchange.SMART,
symbol="goog",
interval=Interval.MINUTE,
start=datetime.datetime(2019, 7, 9, 9),
end=datetime.datetime(2019, 8, 16, 4))
bar_data = rq.rqdata_client.query_history(req)
print(bar_data)
def data_record(start, end, vt_symbol):
from vnpy.trader.database import database_manager
from vnpy.gateway.ib.ib_gateway import IbGateway
from vnpy.trader.utility import load_json
from vnpy.trader.object import HistoryRequest
from vnpy.trader.constant import Interval, Exchange
from dateutil import parser
from vnpy.event.engine import EventEngine
from vnpy.trader.event import EVENT_LOG
vt_symbol = vt_symbol
symbol, exchange = vt_symbol.split('.')
if not start and not end:
offset = 0 if dt.datetime.now().time() > dt.time(17, 0) else 1
start = (dt.datetime.today() - dt.timedelta(days=offset + 1)).replace(
hour=17, minute=0, second=0, microsecond=0)
end = (dt.datetime.today() - dt.timedelta(days=offset)).replace(
hour=17, minute=0, second=0, microsecond=0)
else:
start = parser.parse(start)
end = parser.parse(end) if end else end
ib_settings = load_json('connect_ib.json')
ib_settings["客户号"] += 4
recorder_engine = EventEngine()
def log(event):
data = event.data
print(data.level, data.msg)
recorder_engine.register(EVENT_LOG, log)
ib = IbGateway(recorder_engine)
try:
recorder_engine.start()
ib.connect(ib_settings)
if ib.api.client.isConnected():
req = HistoryRequest(symbol, Exchange(exchange), start, end,
Interval.MINUTE)
ib.write_log(f'发起请求#{vt_symbol}, {start}至{end}')
his_data = ib.query_history(req)
ib.write_log(
f'获得数据#{vt_symbol}， {his_data[0].datetime}至{his_data[-1].datetime}, 共{len(his_data)}条'
)
database_manager.save_bar_data(his_data)
ib.write_log(f'成功入库')
else:
ib.write_log('连接失败！请检查客户号是否被占用或IP是否正确')
except Exception as e:
raise e
finally:
ib.close()
recorder_engine.stop()
def load_history_data(self, req: HistoryRequest):
""""""
self.output("开始加载历史数据")
if not req.end:
req.end = datetime.now()
if req.start >= req.end:
self.output("起始日期必须小于结束日期")
return
self.history_data.clear() # Clear previously loaded history data
# Load 30 days of data each time and allow for progress update
total_days = (req.end - req.start).days
progress_days = max(int(total_days / 10), 1)
progress_delta = timedelta(days=progress_days)
interval_delta = INTERVAL_DELTA_MAP[req.interval]
start = req.start
end = req.start + progress_delta
progress = 0
while start < req.end:
progress_bar = "#" * int(progress * 10 + 1)
self.output(f"加载进度:{progress_bar} [{progress:.0%}]")
end = min(end, req.end) # Make sure end time stays within set range
if self.mode == BacktestingMode.BAR:
data = load_bar_data(
req.symbol,
req.exchange,
req.interval,
start,
end
)
else:
data = load_tick_data(
req.symbol,
req.exchange,
start,
end
)
self.history_data.extend(data)
progress += progress_days / total_days
progress = min(progress, 1)
start = end + interval_delta
end += progress_delta
self.output(f"{req.start}至{req.end}历史数据加载完成，数据量:{len(self.history_data)}")
def diff():
req = HistoryRequest(
symbol=symbol,
exchange=exchange,
interval=Interval.MINUTE,
start=start_date,
end=end_date
)
data = rqdata_client.query_history(req)
print(data)
def write():
req = HistoryRequest(
symbol=symbol,
exchange=exchange,
interval=Interval.MINUTE,
start=start_date,
end=end_date
)
data = rqdata_client.query_history(req)
database_manager.save_bar_data(data)
def main():
""""""
qapp = create_qapp()
event_engine = EventEngine()
main_engine = MainEngine(event_engine)
main_engine.add_gateway(BinancesGateway)
engine_name = 'BINANCES'
setting = {
"key":
"bEZCKVAkJYugYErdlg5GAj7PGgh72ieFIVtDVYgUwJuE7M0vLFM7lN7JuQPYB0AL",
"secret":
"LGZXmPUe4iXqOWlmjiUHGvf7XsdLSBWe5HfXauJoxu8qR9RTeZMUqzTOhcRiAK9K",
"会话数": 3,
"服务器": "REAL",
"合约模式": "正向",
"代理地址": "",
"代理端口": 0
}
main_engine.connect(setting, engine_name)
yearlists = {'2020': '2021'}
# all existed currency futures
symbols = SYMBOLS
# symbol = symbols[0]
# quote = SubscribeRequest(symbol, BinanceGateway.exchanges[0])
# df = main_engine.subscribe(quote,'BINANCE')
INTERV = 'MINUTE'
# INTERV = 'YEAR'
for symbol in symbols:
this_directory = 'C:\\Workspace\\crypto\\data\\' + symbol
if not os.path.exists(this_directory):
os.makedirs(this_directory)
for key, var in yearlists.items():
startdate = pd.to_datetime(key + '-01-01')
enddate = pd.to_datetime(var + '-01-06')
print('query {} minute history '.format(symbol))
quote = HistoryRequest(symbol,
BinancesGateway.exchanges[0],
start=startdate,
end=enddate,
interval=Interval.MINUTE)
df = main_engine.query_history(quote, engine_name)
# HOUR
this_model_file = this_directory + '\\' + symbol + '_FUTURES' + '_' + INTERV + '_' + key + '.pkl'
# MINUTE
# this_model_file = this_directory + '\\' + symbol + '_' + INTERV + '_all' + '.pkl'
with open(this_model_file, 'wb') as f:
# first save model
pickle.dump(df, f)
def query_bar_from_rq(self, symbol: str, exchange: Exchange,
interval: Interval, start: datetime, end: datetime):
"""
Query bar data from RQData.
"""
req = HistoryRequest(symbol=symbol,
exchange=exchange,
interval=interval,
start=start,
end=end)
data = rqdata_client.query_history(req)
return data
def test3():
event_engine = EventEngine()
gate = HbdmGateway(event_engine)
setting = {
"API Key": "mjlpdje3ld-701b9727-9ae52956-3ef1e",
"Secret Key": "b4df4824-124ec192-55786af1-a397a",
"会话数": 3,
"代理地址": "127.0.0.1",
"代理端口": "1080",
}
gate.connect(setting)
time.sleep(3)
print(symbol_type_map)
if len(symbol_type_map) <= 0:
return
req = HistoryRequest(exchange=Exchange.HUOBI,
symbol="BCH190830",
interval=Interval.MINUTE,
start=datetime.datetime(2019, 8, 1, 0),
end=datetime.datetime(2019, 8, 17, 16))
bar_data = gate.query_history(req)
tb = TestBar()
date = []
for v in bar_data:
tb.am.update_bar(v)
date.append(v.datetime)
dt = date
close = tb.am.close_array
rect1 = [0.14, 0.35, 0.77, 0.6] # [左, 下, 宽, 高] 规定的矩形区域 （全部是0~1之间的数，表示比例）
rect2 = [0.14, 0.05, 0.77, 0.2]
fig = plt.figure()
ax1 = plt.axes(rect1)
ax2 = plt.axes(rect2)
# ax1 = fig.add_subplot(2,1,1)
print(dt)
print(close)
ax1.plot(dt, close)
v_list, p_list = tb.am.wave()
for i in range(len(p_list)):
p = p_list[i]
ax1.plot([dt[p]], [close[p]], 'o')
ax1.annotate(close[p], xy=(dt[p], close[p]))
print(close)
sma120 = talib.SMA(close, timeperiod=120)
ax1.plot(dt, sma120)
# ax2 = fig.add_subplot(2,1,2)
kdj_data = tb.am.kdj()
ax2.plot(dt, kdj_data["k"], "r")
ax2.plot(dt, kdj_data["d"], "b")
def query_bar_from_datafeed(self, symbol: str, exchange: Exchange,
interval: Interval, start: datetime,
end: datetime):
"""
Query bar data from datafeed.
"""
req = HistoryRequest(symbol=symbol,
exchange=exchange,
interval=interval,
start=start,
end=end)
data = self.datafeed.query_bar_history(req)
return data
def query_bar_from_tq(
self, symbol: str, exchange: Exchange, interval: Interval, start: datetime, end: datetime
):
req = HistoryRequest(
symbol=symbol,
exchange=exchange,
interval=interval,
start=start,
end=end
)
tqdata_api = TqdataApi()
data = tqdata_api.query_history(req)
return data
def get_historical_data(self, contract, end, bar_count, interval):
total_minutes = {
Interval.MINUTE: 1,
Interval.HOUR: 60
}[interval] * bar_count
start = (end if end else dt.datetime.now(
get_localzone())) - dt.timedelta(minutes=total_minutes)
req = HistoryRequest(contract.symbol,
contract.exchange,
start=start,
end=end,
interval=interval)
his_data = self.main_engine.query_history(req, contract.gateway_name)
return his_data
def download_history_data(symbol, exchange):
print(symbol, exchange)
begin_date = datetime.strptime("2019-09-19", "%Y-%m-%d")
for i in range(1):
start_date = begin_date.replace(year=begin_date.year + i)
# end_date = begin_date.replace(year=begin_date.year + i + 1)
end_date = datetime.now()
print(start_date, end_date)
req = HistoryRequest(symbol=symbol,
exchange=Exchange(exchange),
interval=Interval("1m"),
start=start_date,
end=end_date)
data = mddata_client.query_history(req)
def downloading_history_data(args):
'''
download stocks or futures bar data.
'''
req = HistoryRequest(symbol=args.symbol,
exchange=Exchange(args.exchange),
interval=Interval(args.interval),
start=datetime.strptime(args.startdate, '%Y-%m-%d'),
end=datetime.strptime(args.enddate, '%Y-%m-%d'))
event_engine = EventEngine()
main_engine = MainEngine(event_engine)
main_engine.add_gateway(CtpGateway)
cta_backtester_engine = main_engine.add_app(CtaBacktesterApp)
cta_backtester_engine.start_downloading(req.vt_symbol, req.interval,
req.start, req.end)
def run_downloading(self, vt_symbol: str, interval: str, start: datetime,
end: datetime):
"""
Query bar data from RQData.
"""
self.write_log(f"{vt_symbol}-{interval}开始下载历史数据")
try:
symbol, exchange = extract_vt_symbol(vt_symbol)
except ValueError:
self.write_log(f"{vt_symbol}解析失败，请检查交易所后缀")
self.thread = None
return
req = HistoryRequest(symbol=symbol,
exchange=exchange,
interval=Interval(interval),
start=start,
end=end)
contract = self.main_engine.get_contract(vt_symbol)
try:
# If history data provided in gateway, then query
if contract and contract.history_data:
data = self.main_engine.query_history(req,
contract.gateway_name)
# Otherwise use RQData to query data
else:
if SETTINGS["rqdata.username"]:
data = rqdata_client.query_history(req)
elif SETTINGS["tqdata.username"]:
data = tqdata_client.query_history(req)
else:
data = []
if data:
database_manager.save_bar_data(data)
self.write_log(f"{vt_symbol}-{interval}历史数据下载完成")
else:
self.write_log(f"数据下载失败，无法获取{vt_symbol}的历史数据")
except Exception:
msg = f"数据下载失败，触发异常：\n{traceback.format_exc()}"
self.write_log(msg)
# Clear thread object handler.
self.thread = None
def get_jq_bars(self, symbol: str, interval: str, count: int):
""""""
contract = self.main_engine.get_current_contract()
if not contract:
return []
req = HistoryRequest(
symbol=contract.symbol,
count=count,
exchange=contract.exchange,
interval=Interval(interval),
start="",
end="",
)
bars_list, bars_df = jqdata_client.query_history(req)
return bars_list
def query_bar_from_rq(self, symbol: str, exchange: Exchange,
interval: Interval, start: datetime, end: datetime):
"""
Query bar data from RQData.
"""
req = HistoryRequest(symbol=symbol,
exchange=exchange,
interval=interval,
start=start,
end=end)
# data = mddata_client.query_history(req)
try:
data = mddata_client.query_history(req)
except Exception as ex:
self.write_log(f"{symbol}.{exchange.value}合约下载失败：{ex.args}")
return None
return data
