def load_bar_data(spread: SpreadData,
interval: Interval,
start: datetime,
end: datetime,
pricetick: float = 0):
""""""
# Load bar data of each spread leg
leg_bars: Dict[str, Dict] = {}
for vt_symbol in spread.legs.keys():
symbol, exchange = extract_vt_symbol(vt_symbol)
bar_data: List[BarData] = database_manager.load_bar_data(
symbol, exchange, interval, start, end)
bars: Dict[datetime, BarData] = {bar.datetime: bar for bar in bar_data}
leg_bars[vt_symbol] = bars
# Calculate spread bar data
spread_bars: List[BarData] = []
for dt in bars.keys():
spread_price = 0
spread_value = 0
spread_available = True
for leg in spread.legs.values():
leg_bar = leg_bars[leg.vt_symbol].get(dt, None)
if leg_bar:
price_multiplier = spread.price_multipliers[leg.vt_symbol]
spread_price += price_multiplier * leg_bar.close_price
spread_value += abs(price_multiplier) * leg_bar.close_price
else:
spread_available = False
if spread_available:
if pricetick:
spread_price = round_to(spread_price, pricetick)
spread_bar = BarData(
symbol=spread.name,
exchange=exchange.LOCAL,
datetime=dt,
interval=interval,
open_price=spread_price,
high_price=spread_price,
low_price=spread_price,
close_price=spread_price,
gateway_name="SPREAD",
)
spread_bar.value = spread_value
spread_bars.append(spread_bar)
return spread_bars
def new_bars(self, dt: datetime) -> None:
""""""
self.datetime = dt
# self.bars.clear()
for vt_symbol in self.vt_symbols:
bar = self.history_data.get((dt, vt_symbol), None)
# If bar data of vt_symbol at dt exists
if bar:
self.bars[vt_symbol] = bar
# Otherwise, use previous data to backfill
elif vt_symbol in self.bars:
old_bar = self.bars[vt_symbol]
bar = BarData(symbol=old_bar.symbol,
exchange=old_bar.exchange,
datetime=dt,
open_price=old_bar.close_price,
high_price=old_bar.close_price,
low_price=old_bar.close_price,
close_price=old_bar.close_price,
gateway_name=old_bar.gateway_name)
self.bars[vt_symbol] = bar
self.cross_limit_order()
self.strategy.on_bars(self.bars)
self.update_daily_close(self.bars, dt)
def subscribe(self, req: SubscribeRequest) -> None:
""""""
if req.symbol not in symbol_name_map:
self.gateway.write_log(f"找不到该合约代码{req.symbol}")
return
# Create tick buf data
tick = TickData(
symbol=req.symbol,
name=symbol_name_map.get(req.symbol, ""),
exchange=Exchange.BINANCE,
datetime=datetime.now(LOCAL_TZ),
gateway_name=self.gateway_name,
)
self.ticks[req.symbol.lower()] = tick
# Create bar buf data
bar = BarData(
symbol=req.symbol,
exchange=Exchange.BINANCE,
datetime=datetime.now(LOCAL_TZ),
gateway_name=self.gateway_name,
interval=Interval.MINUTE
)
self.bars[req.symbol.lower()] = bar
# Close previous connection
if self._active:
self.stop()
self.join()
# Create new connection
channels = []
for ws_symbol in self.ticks.keys():
channels.append(ws_symbol + "@ticker")
channels.append(ws_symbol + "@depth5")
channels.append(ws_symbol + "@kline_1m")
if self.server == "REAL":
url = F_WEBSOCKET_DATA_HOST + "/".join(channels)
if not self.usdt_base:
url = D_WEBSOCKET_DATA_HOST + "/".join(channels)
else:
url = F_TESTNET_WEBSOCKET_DATA_HOST + "/".join(channels)
if not self.usdt_base:
url = D_TESTNET_WEBSOCKET_DATA_HOST + "/".join(channels)
self.init(url, self.proxy_host, self.proxy_port)
self.start()
def import_data_from_csv(self, file_path: str, symbol: str,
exchange: Exchange, interval: Interval,
datetime_head: str, open_head: str,
high_head: str, low_head: str, close_head: str,
volume_head: str, open_interest_head: str,
datetime_format: str) -> Tuple:
""""""
with open(file_path, "rt") as f:
buf = [line.replace("\0", "") for line in f]
reader = csv.DictReader(buf, delimiter=",")
bars = []
start = None
count = 0
for item in reader:
if datetime_format:
dt = datetime.strptime(item[datetime_head], datetime_format)
else:
dt = datetime.fromisoformat(item[datetime_head])
open_interest = item.get(open_interest_head, 0)
bar = BarData(
symbol=symbol,
exchange=exchange,
datetime=dt,
interval=interval,
volume=float(item[volume_head]),
open_price=float(item[open_head]),
high_price=float(item[high_head]),
low_price=float(item[low_head]),
close_price=float(item[close_head]),
open_interest=float(open_interest),
gateway_name="DB",
)
bars.append(bar)
# do some statistics
count += 1
if not start:
start = bar.datetime
# insert into database
database_manager.save_bar_data(bars)
end = bar.datetime
return start, end, count
def to_bar(self):
"""
Generate BarData object from DbBarData.
"""
bar = BarData(
symbol=self.symbol,
exchange=Exchange(self.exchange),
datetime=self.datetime.replace(tzinfo=DB_TZ),
interval=Interval(self.interval),
volume=self.volume,
open_price=self.open_price,
high_price=self.high_price,
open_interest=self.open_interest,
low_price=self.low_price,
close_price=self.close_price,
gateway_name="DB",
)
return bar
def load_bars(self, strategy: StrategyTemplate, days: int, interval: Interval):
""""""
vt_symbols = strategy.vt_symbols
dts: Set[datetime] = set()
history_data: Dict[Tuple, BarData] = {}
# Load data from rqdata/gateway/database
for vt_symbol in vt_symbols:
data = self.load_bar(vt_symbol, days, interval)
for bar in data:
dts.add(bar.datetime)
history_data[(bar.datetime, vt_symbol)] = bar
# Convert data structure and push to strategy
dts = list(dts)
dts.sort()
bars = {}
for dt in dts:
for vt_symbol in vt_symbols:
bar = history_data.get((dt, vt_symbol), None)
# If bar data of vt_symbol at dt exists
if bar:
bars[vt_symbol] = bar
# Otherwise, use previous data to backfill
elif vt_symbol in bars:
old_bar = bars[vt_symbol]
bar = BarData(
symbol=old_bar.symbol,
exchange=old_bar.exchange,
datetime=dt,
open_price=old_bar.close_price,
high_price=old_bar.close_price,
low_price=old_bar.close_price,
close_price=old_bar.close_price,
gateway_name=old_bar.gateway_name
)
bars[vt_symbol] = bar
self.call_strategy_func(strategy, strategy.on_bars, bars)
def query_history(self, req: HistoryRequest) -> List[BarData]:
""""""
history = []
limit = 1500
start_time = int(datetime.timestamp(req.start))
while True:
# Create query params
params = {
"symbol": req.symbol,
"interval": INTERVAL_VT2BINANCES[req.interval],
"limit": limit,
"startTime": start_time * 1000, # convert to millisecond
}
# Add end time if specified
if req.end:
end_time = int(datetime.timestamp(req.end))
params["endTime"] = end_time * 1000 # convert to millisecond
# Get response from server
if self.usdt_base:
path = "/fapi/v1/klines"
else:
path = "/dapi/v1/klines"
resp = self.request(
"GET",
path=path,
data={"security": Security.NONE},
params=params
)
# Break if request failed with other status code
if resp.status_code // 100 != 2:
msg = f"获取历史数据失败，状态码：{resp.status_code}，信息：{resp.text}"
self.gateway.write_log(msg)
break
else:
data = resp.json()
if not data:
msg = f"获取历史数据为空，开始时间：{start_time}"
self.gateway.write_log(msg)
break
buf = []
for l in data:
bar = BarData(
symbol=req.symbol,
exchange=req.exchange,
datetime=generate_datetime(l[0]),
interval=req.interval,
volume=float(l[5]),
open_price=float(l[1]),
high_price=float(l[2]),
low_price=float(l[3]),
close_price=float(l[4]),
gateway_name=self.gateway_name
)
buf.append(bar)
history.extend(buf)
begin = buf[0].datetime
end = buf[-1].datetime
msg = f"获取历史数据成功，{req.symbol} - {req.interval.value}，{begin} - {end}"
self.gateway.write_log(msg)
# Break if total data count less than limit (latest date collected)
if len(data) < limit:
break
# Update start time
start_dt = bar.datetime + TIMEDELTA_MAP[req.interval]
start_time = int(datetime.timestamp(start_dt))
return history
def get_binance_data(symbol: str, exchanges: str, start_time: str, end_time: str):
"""
爬取币安交易所的数据
:param symbol: BTCUSDT.
:param exchanges: 现货、USDT合约, 或者币币合约.
:param start_time: 格式如下:2020-1-1 或者2020-01-01
:param end_time: 格式如下:2020-1-1 或者2020-01-01
:return:
"""
api_url = ''
save_symbol = symbol
gate_way = 'BINANCES'
if exchanges == 'spot':
print("spot")
limit = BINANCE_SPOT_LIMIT
save_symbol = symbol.lower()
gate_way = 'BINANCE'
api_url = f'https://api.binance.com/api/v3/klines?symbol={symbol}&interval=1m&limit={limit}'
elif exchanges == 'future':
print('future')
limit = BINANCE_FUTURE_LIMIT
api_url = f'https://fapi.binance.com/fapi/v1/klines?symbol={symbol}&interval=1m&limit={limit}'
elif exchanges == 'coin_future':
print("coin_future")
limit = BINANCE_FUTURE_LIMIT
f'https://dapi.binance.com/dapi/v1/klines?symbol={symbol}&interval=1m&limit={limit}'
else:
raise Exception('交易所名称请输入以下其中一个：spot, future, coin_future')
start_time = int(datetime.strptime(start_time, '%Y-%m-%d').timestamp() * 1000)
end_time = int(datetime.strptime(end_time, '%Y-%m-%d').timestamp() * 1000)
while True:
try:
print(start_time)
url = f'{api_url}&startTime={start_time}'
print(url)
data = requests.get(url=url, timeout=10, proxies=proxies).json()
"""
[
[
1591258320000, // 开盘时间
"9640.7", // 开盘价
"9642.4", // 最高价
"9640.6", // 最低价
"9642.0", // 收盘价(当前K线未结束的即为最新价)
"206", // 成交量
1591258379999, // 收盘时间
"2.13660389", // 成交额(标的数量)
48, // 成交笔数
"119", // 主动买入成交量
"1.23424865", // 主动买入成交额(标的数量)
"0" // 请忽略该参数
]
"""
buf = []
for l in data:
bar = BarData(
symbol=save_symbol,
exchange=Exchange.BINANCE,
datetime=generate_datetime(l[0]),
interval=Interval.MINUTE,
volume=float(l[5]),
open_price=float(l[1]),
high_price=float(l[2]),
low_price=float(l[3]),
close_price=float(l[4]),
gateway_name=gate_way
)
buf.append(bar)
database_manager.save_bar_data(buf)
# 到结束时间就退出, 后者收盘价大于当前的时间.
if (data[-1][0] > end_time) or data[-1][6] >= (int(time.time() * 1000) - 60 * 1000):
break
start_time = data[-1][0]
except Exception as error:
print(error)
time.sleep(10)