def load_bar_data(spread: SpreadData,
interval: Interval,
start: datetime,
end: datetime,
pricetick: float = 0):
""""""
# Load bar data of each spread leg
leg_bars: Dict[str, Dict] = {}
for vt_symbol in spread.legs.keys():
symbol, exchange = extract_vt_symbol(vt_symbol)
bar_data: List[BarData] = database_manager.load_bar_data(
symbol, exchange, interval, start, end)
bars: Dict[datetime, BarData] = {bar.datetime: bar for bar in bar_data}
leg_bars[vt_symbol] = bars
# Calculate spread bar data
spread_bars: List[BarData] = []
for dt in bars.keys():
spread_price = 0
spread_value = 0
spread_available = True
for leg in spread.legs.values():
leg_bar = leg_bars[leg.vt_symbol].get(dt, None)
if leg_bar:
price_multiplier = spread.price_multipliers[leg.vt_symbol]
spread_price += price_multiplier * leg_bar.close_price
spread_value += abs(price_multiplier) * leg_bar.close_price
else:
spread_available = False
if spread_available:
if pricetick:
spread_price = round_to(spread_price, pricetick)
spread_bar = BarData(
symbol=spread.name,
exchange=exchange.LOCAL,
datetime=dt,
interval=interval,
open_price=spread_price,
high_price=spread_price,
low_price=spread_price,
close_price=spread_price,
gateway_name="SPREAD",
)
spread_bar.value = spread_value
spread_bars.append(spread_bar)
return spread_bars
def query_history(self, req: HistoryRequest):
""""""
buf = {}
end_time = None
for i in range(10):
path = f"/api/spot/v3/instruments/{req.symbol}/candles"
# Create query params
params = {"granularity": INTERVAL_VT2OKEX[req.interval]}
if end_time:
end = datetime.strptime(end_time, "%Y-%m-%dT%H:%M:%S.%fZ")
start = end - TIMEDELTA_MAP[req.interval] * 200
params["start"] = start.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
params["end"] = end.strftime("%Y-%m-%dT%H:%M:%S.%fZ")
# Get response from server
resp = self.request("GET", path, params=params)
# Break if request failed with other status code
if resp.status_code // 100 != 2:
msg = f"获取历史数据失败，状态码：{resp.status_code}，信息：{resp.text}"
self.gateway.write_log(msg)
break
else:
data = resp.json()
if not data:
msg = f"获取历史数据为空"
break
for l in data:
ts, o, h, l, c, v = l
dt = generate_datetime(ts)
bar = BarData(symbol=req.symbol,
exchange=req.exchange,
datetime=dt,
interval=req.interval,
volume=float(v),
open_price=float(o),
high_price=float(h),
low_price=float(l),
close_price=float(c),
gateway_name=self.gateway_name)
buf[bar.datetime] = bar
begin = data[-1][0]
end = data[0][0]
msg = f"获取历史数据成功，{req.symbol} - {req.interval.value}，{begin} - {end}"
self.gateway.write_log(msg)
# Update start time
end_time = begin
index = list(buf.keys())
index.sort()
history = [buf[i] for i in index]
return history
def new_bars(self, dt: datetime) -> None:
""""""
self.datetime = dt
# self.bars.clear()
for vt_symbol in self.vt_symbols:
bar = self.history_data.get((dt, vt_symbol), None)
# If bar data of vt_symbol at dt exists
if bar:
self.bars[vt_symbol] = bar
# Otherwise, use previous data to backfill
elif vt_symbol in self.bars:
old_bar = self.bars[vt_symbol]
bar = BarData(symbol=old_bar.symbol,
exchange=old_bar.exchange,
datetime=dt,
open_price=old_bar.close_price,
high_price=old_bar.close_price,
low_price=old_bar.close_price,
close_price=old_bar.close_price,
gateway_name=old_bar.gateway_name)
self.bars[vt_symbol] = bar
self.cross_limit_order()
self.strategy.on_bars(self.bars)
self.update_daily_close(self.bars, dt)
def subscribe(self, req: SubscribeRequest) -> None:
""""""
if req.symbol not in symbol_name_map:
self.gateway.write_log(f"找不到该合约代码{req.symbol}")
return
# Create tick buf data
tick = TickData(
symbol=req.symbol,
name=symbol_name_map.get(req.symbol, ""),
exchange=Exchange.BINANCE,
datetime=datetime.now(CHINA_TZ),
gateway_name=self.gateway_name,
)
self.ticks[req.symbol.lower()] = tick
# Create bar buf data
bar = BarData(
symbol=req.symbol,
exchange=Exchange.BINANCE,
datetime=datetime.now(CHINA_TZ),
gateway_name=self.gateway_name,
interval=Interval.MINUTE
)
self.bars[req.symbol.lower()] = bar
self.last[req.symbol.lower()] = {}
self.trades[req.symbol.lower()] = []
# Close previous connection
if self._active:
self.stop()
self.join()
# Create new connection
channels = []
for ws_symbol in self.ticks.keys():
# channels.append(ws_symbol + "@ticker")
channels.append(ws_symbol + "@depth5@100ms")
channels.append(ws_symbol + "@kline_1m")
channels.append(ws_symbol + "@aggTrade")
if self.server == "REAL":
url = F_WEBSOCKET_DATA_HOST + "/".join(channels)
if not self.usdt_base:
url = D_WEBSOCKET_DATA_HOST + "/".join(channels)
else:
url = F_TESTNET_WEBSOCKET_DATA_HOST + "/".join(channels)
if not self.usdt_base:
url = D_TESTNET_WEBSOCKET_DATA_HOST + "/".join(channels)
self.init(url, self.proxy_host, self.proxy_port)
self.start()
def import_data_from_csv(self, file_path: str, symbol: str,
exchange: Exchange, interval: Interval,
datetime_head: str, open_head: str,
high_head: str, low_head: str, close_head: str,
volume_head: str, open_interest_head: str,
datetime_format: str) -> Tuple:
""""""
with open(file_path, "rt") as f:
buf = [line.replace("\0", "") for line in f]
reader = csv.DictReader(buf, delimiter=",")
bars = []
start = None
count = 0
for item in reader:
if datetime_format:
dt = datetime.strptime(item[datetime_head], datetime_format)
else:
dt = datetime.fromisoformat(item[datetime_head])
open_interest = item.get(open_interest_head, 0)
bar = BarData(
symbol=symbol,
exchange=exchange,
datetime=dt,
interval=interval,
volume=float(item[volume_head]),
open_price=float(item[open_head]),
high_price=float(item[high_head]),
low_price=float(item[low_head]),
close_price=float(item[close_head]),
open_interest=float(open_interest),
gateway_name="DB",
)
bars.append(bar)
# do some statistics
count += 1
if not start:
start = bar.datetime
# insert into database
database_manager.save_bar_data(bars)
end = bar.datetime
return start, end, count
def query_history(self, req: HistoryRequest):
""""""
history = []
interval = INTERVAL_VT2GATEIO[req.interval]
params = {
"contract": req.symbol,
"limit": 2000,
"interval": interval,
}
resp = self.request(
method="GET",
path="/api/v4/futures/usdt/candlesticks",
params=params
)
if resp.status_code // 100 != 2:
msg = f"获取历史数据失败，状态码：{resp.status_code}，信息：{resp.text}"
self.gateway.write_log(msg)
else:
data = resp.json()
if not data:
msg = f"获取历史数据为空"
for d in data:
bar = BarData(
symbol=req.symbol,
exchange=req.exchange,
datetime=generate_datetime(d["t"]),
interval=req.interval,
volume=d["v"],
open_price=float(d["o"]),
high_price=float(d["h"]),
low_price=float(d["l"]),
close_price=float(d["c"]),
gateway_name=self.gateway_name
)
history.append(bar)
begin = generate_datetime(data[0]["t"])
end = generate_datetime(data[-1]["t"])
msg = f"获取历史数据成功，{req.symbol} - {req.interval.value}，{begin} - {end}"
self.gateway.write_log(msg)
return history
def query_history(self, req: HistoryRequest) -> List[BarData]:
""""""
# Create query params
params = {
"symbol": req.symbol,
"period": INTERVAL_VT2HUOBI[req.interval],
"size": 2000
}
# Get response from server
resp = self.request("GET", "/market/history/kline", params=params)
# Break if request failed with other status code
history = []
if resp.status_code // 100 != 2:
msg = f"获取历史数据失败，状态码：{resp.status_code}，信息：{resp.text}"
self.gateway.write_log(msg)
else:
data = resp.json()
if not data:
msg = f"获取历史数据为空"
self.gateway.write_log(msg)
else:
for d in data["data"]:
dt = generate_datetime(d["id"])
bar = BarData(symbol=req.symbol,
exchange=req.exchange,
datetime=dt,
interval=req.interval,
volume=d["vol"],
open_price=d["open"],
high_price=d["high"],
low_price=d["low"],
close_price=d["close"],
gateway_name=self.gateway_name)
history.append(bar)
begin = history[0].datetime
end = history[-1].datetime
msg = f"获取历史数据成功，{req.symbol} - {req.interval.value}，{begin} - {end}"
self.gateway.write_log(msg)
return history
def to_bar(self):
"""
Generate BarData object from DbBarData.
"""
bar = BarData(
symbol=self.symbol,
exchange=Exchange(self.exchange),
datetime=self.datetime.replace(tzinfo=DB_TZ),
interval=Interval(self.interval),
volume=self.volume,
open_price=self.open_price,
high_price=self.high_price,
open_interest=self.open_interest,
low_price=self.low_price,
close_price=self.close_price,
gateway_name="DB",
)
return bar
def load_bars(self, strategy: StrategyTemplate, days: int,
interval: Interval):
""""""
vt_symbols = strategy.vt_symbols
dts: Set[datetime] = set()
history_data: Dict[Tuple, BarData] = {}
# Load data from rqdata/gateway/database
for vt_symbol in vt_symbols:
data = self.load_bar(vt_symbol, days, interval)
for bar in data:
dts.add(bar.datetime)
history_data[(bar.datetime, vt_symbol)] = bar
# Convert data structure and push to strategy
dts = list(dts)
dts.sort()
bars = {}
for dt in dts:
for vt_symbol in vt_symbols:
bar = history_data.get((dt, vt_symbol), None)
# If bar data of vt_symbol at dt exists
if bar:
bars[vt_symbol] = bar
# Otherwise, use previous data to backfill
elif vt_symbol in bars:
old_bar = bars[vt_symbol]
bar = BarData(symbol=old_bar.symbol,
exchange=old_bar.exchange,
datetime=dt,
open_price=old_bar.close_price,
high_price=old_bar.close_price,
low_price=old_bar.close_price,
close_price=old_bar.close_price,
gateway_name=old_bar.gateway_name)
bars[vt_symbol] = bar
self.call_strategy_func(strategy, strategy.on_bars, bars)
def subscribe(self, req: SubscribeRequest):
""""""
if req.symbol not in symbol_name_map:
self.gateway.write_log(f"找不到该合约代码{req.symbol}")
return
# Create tick buf data
tick = TickData(
symbol=req.symbol,
name=symbol_name_map.get(req.symbol, ""),
exchange=Exchange.BINANCE,
datetime=datetime.now(CHINA_TZ),
gateway_name=self.gateway_name,
)
self.ticks[req.symbol] = tick
# Create bar buf data
bar = BarData(
symbol=req.symbol,
exchange=Exchange.BINANCE,
datetime=datetime.now(CHINA_TZ),
gateway_name=self.gateway_name,
interval=Interval.MINUTE
)
self.bars[req.symbol] = bar
# Close previous connection
if self._active:
self.stop()
self.join()
# Create new connection
channels = []
for ws_symbol in self.ticks.keys():
channels.append(ws_symbol + "@ticker")
channels.append(ws_symbol + "@depth5")
channels.append(ws_symbol + "@kline_1m")
url = WEBSOCKET_DATA_HOST + "/".join(channels)
self.init(url, self.proxy_host, self.proxy_port)
self.start()
def get_binance_data(symbol: str, exchanges: str, start_time: str,
end_time: str):
"""
爬取币安交易所的数据
:param symbol: BTCUSDT.
:param exchanges: 现货、USDT合约, 或者币币合约.
:param start_time: 格式如下:2020-1-1 或者2020-01-01
:param end_time: 格式如下:2020-1-1 或者2020-01-01
:return:
"""
api_url = ''
save_symbol = symbol
gate_way = 'BINANCES'
if exchanges == 'spot':
print("spot")
limit = BINANCE_SPOT_LIMIT
save_symbol = symbol.lower()
gate_way = 'BINANCE'
api_url = f'https://api.binance.com/api/v3/klines?symbol={symbol}&interval=1m&limit={limit}'
elif exchanges == 'future':
print('future')
limit = BINANCE_FUTURE_LIMIT
api_url = f'https://fapi.binance.com/fapi/v1/klines?symbol={symbol}&interval=1m&limit={limit}'
elif exchanges == 'coin_future':
print("coin_future")
limit = BINANCE_FUTURE_LIMIT
f'https://dapi.binance.com/dapi/v1/klines?symbol={symbol}&interval=1m&limit={limit}'
else:
raise Exception('交易所名称请输入以下其中一个：spot, future, coin_future')
start_time = int(
datetime.strptime(start_time, '%Y-%m-%d').timestamp() * 1000)
end_time = int(datetime.strptime(end_time, '%Y-%m-%d').timestamp() * 1000)
while True:
try:
print(start_time)
url = f'{api_url}&startTime={start_time}'
print(url)
data = requests.get(url=url, timeout=10, proxies=proxies).json()
"""
[
[
1591258320000, // 开盘时间
"9640.7", // 开盘价
"9642.4", // 最高价
"9640.6", // 最低价
"9642.0", // 收盘价(当前K线未结束的即为最新价)
"206", // 成交量
1591258379999, // 收盘时间
"2.13660389", // 成交额(标的数量)
48, // 成交笔数
"119", // 主动买入成交量
"1.23424865", // 主动买入成交额(标的数量)
"0" // 请忽略该参数
]
"""
buf = []
for l in data:
bar = BarData(symbol=save_symbol,
exchange=Exchange.BINANCE,
datetime=generate_datetime(l[0]),
interval=Interval.MINUTE,
volume=float(l[5]),
open_price=float(l[1]),
high_price=float(l[2]),
low_price=float(l[3]),
close_price=float(l[4]),
gateway_name=gate_way)
buf.append(bar)
database_manager.save_bar_data(buf)
# 到结束时间就退出, 后者收盘价大于当前的时间.
if (data[-1][0] > end_time
) or data[-1][6] >= (int(time.time() * 1000) - 60 * 1000):
break
start_time = data[-1][0]
except Exception as error:
print(error)
time.sleep(10)
def query_history(self, req: HistoryRequest) -> List[BarData]:
""""""
history = []
limit = 1500
start_time = int(datetime.timestamp(req.start))
while True:
# Create query params
params = {
"symbol": req.symbol,
"interval": INTERVAL_VT2BINANCES[req.interval],
"limit": limit,
"startTime": start_time * 1000, # convert to millisecond
}
# Add end time if specified
if req.end:
end_time = int(datetime.timestamp(req.end))
params["endTime"] = end_time * 1000 # convert to millisecond
# Get response from server
if self.usdt_base:
path = "/fapi/v1/klines"
else:
path = "/dapi/v1/klines"
resp = self.request(
"GET",
path=path,
data={"security": Security.NONE},
params=params
)
# Break if request failed with other status code
if resp.status_code // 100 != 2:
msg = f"获取历史数据失败，状态码：{resp.status_code}，信息：{resp.text}"
self.gateway.write_log(msg)
break
else:
data = resp.json()
if not data:
msg = f"获取历史数据为空，开始时间：{start_time}"
self.gateway.write_log(msg)
break
buf = []
for l in data:
bar = BarData(
symbol=req.symbol,
exchange=req.exchange,
datetime=generate_datetime(l[0]),
interval=req.interval,
volume=float(l[5]),
open_price=float(l[1]),
high_price=float(l[2]),
low_price=float(l[3]),
close_price=float(l[4]),
gateway_name=self.gateway_name
)
buf.append(bar)
history.extend(buf)
begin = buf[0].datetime
end = buf[-1].datetime
msg = f"获取历史数据成功，{req.symbol} - {req.interval.value}，{begin} - {end}"
self.gateway.write_log(msg)
# Break if total data count less than limit (latest date collected)
if len(data) < limit:
break
# Update start time
start_dt = bar.datetime + TIMEDELTA_MAP[req.interval]
start_time = int(datetime.timestamp(start_dt))
return history
def query_history(self, req: HistoryRequest) -> List[BarData]:
""""""
history = []
count = 2000
start = req.start
time_delta = TIMEDELTA_MAP[req.interval]
# Convert symbol
buf = [i for i in req.symbol if not i.isdigit()]
symbol = "".join(buf)
ws_symbol = f"{symbol}"
while True:
# Calculate end time
end = start + time_delta * count
# Create query params
params = {
"symbol": ws_symbol,
"period": INTERVAL_VT2HUOBIS[req.interval],
"from": int(start.timestamp()),
"to": int(end.timestamp())
}
# Get response from server
resp = self.request("GET",
"/swap-ex/market/history/kline",
params=params)
# Break if request failed with other status code
if resp.status_code // 100 != 2:
msg = f"获取历史数据失败，状态码：{resp.status_code}，信息：{resp.text}"
self.gateway.write_log(msg)
break
else:
data = resp.json()
if not data:
msg = f"获取历史数据为空"
self.gateway.write_log(msg)
break
if not data["data"]:
msg = f"获取历史数据为空"
self.gateway.write_log(msg)
break
buf = []
print("download data:", data)
for d in data["data"]:
dt = generate_datetime(d["id"])
bar = BarData(symbol=req.symbol,
exchange=req.exchange,
datetime=dt,
interval=req.interval,
volume=d["vol"],
open_price=d["open"],
high_price=d["high"],
low_price=d["low"],
close_price=d["close"],
gateway_name=self.gateway_name)
buf.append(bar)
history.extend(buf)
begin = buf[0].datetime
end = buf[-1].datetime
msg = f"获取历史数据成功，{req.symbol} - {req.interval.value}，{begin} - {end}"
self.gateway.write_log(msg)
# Update start time
start = bar.datetime
# Break if data end reached
if len(buf) < count:
break
return history