def load_bar_data(
self,
symbol: str,
exchange: Exchange,
interval: Interval,
start: datetime,
end: datetime
) -> List[BarData]:
""""""
s: QuerySet = DbBarData.objects(
symbol=symbol,
exchange=exchange.value,
interval=interval.value,
datetime__gte=convert_tz(start),
datetime__lte=convert_tz(end),
)
vt_symbol = f"{symbol}.{exchange.value}"
bars: List[BarData] = []
for db_bar in s:
db_bar.datetime = DB_TZ.localize(db_bar.datetime)
db_bar.exchange = Exchange(db_bar.exchange)
db_bar.interval = Interval(db_bar.interval)
db_bar.gateway_name = "DB"
db_bar.vt_symbol = vt_symbol
bars.append(db_bar)
return bars
def load_tick_data(
self,
symbol: str,
exchange: Exchange,
start: datetime,
end: datetime
) -> List[TickData]:
""""""
s: QuerySet = DbTickData.objects(
symbol=symbol,
exchange=exchange.value,
datetime__gte=convert_tz(start),
datetime__lte=convert_tz(end),
)
vt_symbol = f"{symbol}.{exchange.value}"
ticks: List[TickData] = []
for db_tick in s:
db_tick.datetime = DB_TZ.localize(db_tick.datetime)
db_tick.exchange = Exchange(db_tick.exchange)
db_tick.gateway_name = "DB"
db_tick.vt_symbol = vt_symbol
ticks.append(db_tick)
return ticks
def load_tick_data(
self,
symbol: str,
exchange: Exchange,
start: datetime,
end: datetime
) -> List[TickData]:
""""""
s: QuerySet = DbTickData.objects(
symbol=symbol,
exchange=exchange.value,
datetime__gte=convert_tz(start),
datetime__lte=convert_tz(end),
)
ticks: List[TickData] = []
for db_tick in s:
tick = TickData(
symbol=db_tick.symbol,
exchange=Exchange(db_tick.exchange),
datetime=db_tick.datetime.astimezone(DB_TZ),
name=db_tick.name,
volume=db_tick.volume,
turnover=db_tick.turnover,
open_interest=db_tick.open_interest,
last_price=db_tick.last_price,
last_volume=db_tick.last_volume,
limit_up=db_tick.limit_up,
limit_down=db_tick.limit_down,
open_price=db_tick.open_price,
high_price=db_tick.high_price,
low_price=db_tick.low_price,
pre_close=db_tick.pre_close,
bid_price_1=db_tick.bid_price_1,
bid_price_2=db_tick.bid_price_2,
bid_price_3=db_tick.bid_price_3,
bid_price_4=db_tick.bid_price_4,
bid_price_5=db_tick.bid_price_5,
ask_price_1=db_tick.ask_price_1,
ask_price_2=db_tick.ask_price_2,
ask_price_3=db_tick.ask_price_3,
ask_price_4=db_tick.ask_price_4,
ask_price_5=db_tick.ask_price_5,
bid_volume_1=db_tick.bid_volume_1,
bid_volume_2=db_tick.bid_volume_2,
bid_volume_3=db_tick.bid_volume_3,
bid_volume_4=db_tick.bid_volume_4,
bid_volume_5=db_tick.bid_volume_5,
ask_volume_1=db_tick.ask_volume_1,
ask_volume_2=db_tick.ask_volume_2,
ask_volume_3=db_tick.ask_volume_3,
ask_volume_4=db_tick.ask_volume_4,
ask_volume_5=db_tick.ask_volume_5,
localtime=db_tick.localtime,
gateway_name="DB"
)
ticks.append(tick)
return ticks
def save_tick_data(self, ticks: List[TickData]) -> bool:
""""""
# Convert bar object to dict and adjust timezone
data = []
for tick in ticks:
tick.datetime = convert_tz(tick.datetime)
d = tick.__dict__
d["exchange"] = d["exchange"].value
d.pop("gateway_name")
d.pop("vt_symbol")
data.append(d)
# Upsert data into database
with self.db.atomic():
for d in data:
DbTickData.insert(d).on_conflict(
update=d,
conflict_target=(
DbTickData.symbol,
DbTickData.exchange,
DbTickData.datetime,
),
).execute()
def save_bar_data(self, bars: List[BarData]) -> bool:
""""""
# Store key parameters
bar = bars[0]
symbol = bar.symbol
exchange = bar.exchange
interval = bar.interval
# Convert bar object to dict and adjust timezone
data = []
for bar in bars:
bar.datetime = convert_tz(bar.datetime)
d = bar.__dict__
d["exchange"] = d["exchange"].value
d["interval"] = d["interval"].value
d.pop("gateway_name")
d.pop("vt_symbol")
data.append(d)
# Upsert data into database
with self.db.atomic():
for d in data:
DbBarData.insert(d).on_conflict(
update=d,
conflict_target=(
DbBarData.symbol,
DbBarData.exchange,
DbBarData.interval,
DbBarData.datetime,
),
).execute()
# Update bar overview
overview: DbBarOverview = DbBarOverview.get_or_none(
DbBarOverview.symbol == symbol,
DbBarOverview.exchange == exchange.value,
DbBarOverview.interval == interval.value,
)
if not overview:
overview = DbBarOverview()
overview.symbol = symbol
overview.exchange = exchange.value
overview.interval = interval.value
overview.start = bars[0].datetime
overview.end = bars[-1].datetime
overview.count = len(bars)
else:
overview.start = min(bars[0].datetime, overview.start)
overview.end = max(bars[-1].datetime, overview.end)
s: ModelSelect = DbBarData.select().where(
(DbBarData.symbol == symbol)
& (DbBarData.exchange == exchange.value)
& (DbBarData.interval == interval.value))
overview.count = s.count()
overview.save()
def save_tick_data(self,
ticks: List[TickData],
collection_name: str = None) -> bool:
""""""
for tick in ticks:
tick.datetime = convert_tz(tick.datetime)
d = tick.__dict__
d["exchange"] = d["exchange"].value
d["interval"] = d["interval"].value
d.pop("gateway_name")
d.pop("vt_symbol")
param = to_update_param(d)
if not collection_name:
DbTickData.objects(
symbol=d["symbol"],
exchange=d["exchange"],
datetime=d["datetime"],
).update_one(upsert=True, **param)
else:
with switch_collection(DbTickData, collection_name):
DbTickData.objects(
symbol=d["symbol"],
exchange=d["exchange"],
datetime=d["datetime"],
).update_one(upsert=True, **param)
def save_tick_data(self, ticks: List[TickData]) -> bool:
""""""
json_body = []
tick = ticks[0]
vt_symbol = tick.vt_symbol
for tick in ticks:
tick.datetime = convert_tz(tick.datetime)
d = {
"measurement": "tick_data",
"tags": {
"vt_symbol": vt_symbol
},
"time": tick.datetime.isoformat(),
"fields": {
"name": tick.name,
"volume": tick.volume,
"open_interest": tick.open_interest,
"last_price": tick.last_price,
"last_volume": tick.last_volume,
"limit_up": tick.limit_up,
"limit_down": tick.limit_down,
"open_price": tick.open_price,
"high_price": tick.high_price,
"low_price": tick.low_price,
"pre_close": tick.pre_close,
"bid_price_1": tick.bid_price_1,
"bid_price_2": tick.bid_price_2,
"bid_price_3": tick.bid_price_3,
"bid_price_4": tick.bid_price_4,
"bid_price_5": tick.bid_price_5,
"ask_price_1": tick.ask_price_1,
"ask_price_2": tick.ask_price_2,
"ask_price_3": tick.ask_price_3,
"ask_price_4": tick.ask_price_4,
"ask_price_5": tick.ask_price_5,
"bid_volume_1": tick.bid_volume_1,
"bid_volume_2": tick.bid_volume_2,
"bid_volume_3": tick.bid_volume_3,
"bid_volume_4": tick.bid_volume_4,
"bid_volume_5": tick.bid_volume_5,
"ask_volume_1": tick.ask_volume_1,
"ask_volume_2": tick.ask_volume_2,
"ask_volume_3": tick.ask_volume_3,
"ask_volume_4": tick.ask_volume_4,
"ask_volume_5": tick.ask_volume_5,
}
}
json_body.append(d)
self.client.write_points(json_body, batch_size=10000)
def save_bar_data(self, bars: List[BarData]) -> bool:
"""保存K线数据"""
# Store key parameters
bar = bars[0]
symbol = bar.symbol
exchange = bar.exchange
interval = bar.interval
# Convert bar object to dict and adjust timezone
data = []
for bar in bars:
bar.datetime = convert_tz(bar.datetime)
data = DbBarData(
symbol=bar.symbol,
exchange=exchange.value,
datetime=bar.datetime,
interval=interval.value,
volume=bar.volume,
open_interest=bar.open_interest,
open_price=bar.open_price,
high_price=bar.high_price,
low_price=bar.low_price,
close_price=bar.close_price
)
self.db.merge(data)
self.db.commit()
overview = self.db.query(DbBarOverview).filter(
DbBarOverview.symbol == symbol,
DbBarOverview.exchange == exchange.value,
DbBarOverview.interval == interval.value
).first()
if not overview:
overview = DbBarOverview(
symbol=symbol,
exchange=exchange.value,
interval=interval.value,
count=len(bars),
start=bars[0].datetime,
end=bars[-1].datetime
)
else:
overview.start = min(bars[0].datetime, overview.start)
overview.end = max(bars[-1].datetime, overview.end)
overview.count = self.db.query(DbBarData).filter(
DbBarData.symbol == symbol,
DbBarData.exchange == exchange.value,
DbBarData.interval == interval.value
).count()
self.db.merge(overview)
self.db.commit()
def save_bar_data(self, bars: List[BarData]) -> bool:
""""""
# Store key parameters
bar = bars[0]
symbol = bar.symbol
exchange = bar.exchange
interval = bar.interval
# Upsert data into mongodb
for bar in bars:
bar.datetime = convert_tz(bar.datetime)
d = bar.__dict__
d["exchange"] = d["exchange"].value
d["interval"] = d["interval"].value
d.pop("gateway_name")
d.pop("vt_symbol")
param = to_update_param(d)
DbBarData.objects(
symbol=d["symbol"],
exchange=d["exchange"],
interval=d["interval"],
datetime=d["datetime"],
).update_one(upsert=True, **param)
# Update bar overview
try:
overview: DbBarOverview = DbBarOverview.objects(
symbol=symbol,
exchange=exchange.value,
interval=interval.value
).get()
except DoesNotExist:
overview: DbBarOverview = DbBarOverview(
symbol=symbol,
exchange=exchange.value,
interval=interval.value
)
if not overview.start:
overview.start = bars[0].datetime
overview.end = bars[-1].datetime
overview.count = len(bars)
else:
overview.start = min(bars[0].datetime, overview.start)
overview.end = max(bars[-1].datetime, overview.end)
overview.count = DbBarData.objects(
symbol=symbol,
exchange=exchange.value,
interval=interval.value
).count()
overview.save()
def load_bar_data(
self,
symbol: str,
exchange: Exchange,
interval: Interval,
start: datetime,
end: datetime
) -> List[BarData]:
""""""
s: QuerySet = DbBarData.objects(
symbol=symbol,
exchange=exchange.value,
interval=interval.value,
datetime__gte=convert_tz(start),
datetime__lte=convert_tz(end),
)
bars: List[BarData] = []
for db_bar in s:
bar = BarData(
symbol=db_bar.symbol,
exchange=Exchange(db_bar.exchange),
datetime=db_bar.datetime.astimezone(DB_TZ),
interval=Interval(db_bar.interval),
volume=db_bar.volume,
turnover=db_bar.turnover,
open_interest=db_bar.open_interest,
open_price=db_bar.open_price,
high_price=db_bar.high_price,
low_price=db_bar.low_price,
close_price=db_bar.close_price,
gateway_name="DB"
)
bars.append(bar)
return bars
def save_tick_data(self, ticks: List[TickData]) -> bool:
""""""
# Convert tick object to dict and adjust timezone
tick = ticks[0]
exchange = tick.exchange
data = []
for tick in ticks:
tick.datetime = convert_tz(tick.datetime)
data = DbTickData(
symbol=tick.symbol,
exchange=exchange.value,
datetime=tick.datetime,
volume=tick.volume,
open_interest=tick.open_interest,
last_price=tick.last_price,
last_volume=tick.last_volume,
limit_up=tick.limit_up,
limit_down=tick.limit_down,
open_price=tick.open_price,
high_price=tick.high_price,
low_price=tick.low_price,
pre_close=tick.pre_close,
bid_price_1=tick.bid_price_1,
bid_price_2=tick.bid_price_2,
bid_price_3=tick.bid_price_3,
bid_price_4=tick.bid_price_4,
bid_price_5=tick.bid_price_5,
ask_price_1=tick.ask_price_1,
ask_price_2=tick.ask_price_2,
ask_price_3=tick.ask_price_3,
ask_price_4=tick.ask_price_4,
ask_price_5=tick.ask_price_5,
bid_volume_1=tick.bid_volume_1,
bid_volume_2=tick.bid_volume_2,
bid_volume_3=tick.bid_volume_3,
bid_volume_4=tick.bid_volume_4,
bid_volume_5=tick.bid_volume_5,
ask_volume_1=tick.ask_volume_1,
ask_volume_2=tick.ask_volume_2,
ask_volume_3=tick.ask_volume_3,
ask_volume_4=tick.ask_volume_4,
ask_volume_5=tick.ask_volume_5
)
self.db.merge(data)
self.db.commit()
def save_tick_data(self, ticks: List[TickData]) -> bool:
""""""
for tick in ticks:
tick.datetime = convert_tz(tick.datetime)
d = tick.__dict__
d["exchange"] = d["exchange"].value
d.pop("gateway_name")
d.pop("vt_symbol")
param = to_update_param(d)
DbTickData.objects(
symbol=d["symbol"],
exchange=d["exchange"],
datetime=d["datetime"],
).update_one(upsert=True, **param)
def save_tick_data(self, ticks: List[TickData]) -> bool:
""""""
# Convert bar object to dict and adjust timezone
data = []
for tick in ticks:
tick.datetime = convert_tz(tick.datetime)
d = tick.__dict__
d["exchange"] = d["exchange"].value
d.pop("gateway_name")
d.pop("vt_symbol")
data.append(d)
# Upsert data into database
with self.db.atomic():
for c in chunked(data, 50):
DbTickData.insert_many(c).on_conflict_replace().execute()
def save_bar_data(self, bars: List[BarData]) -> bool:
""""""
json_body = []
bar = bars[0]
vt_symbol = bar.vt_symbol
interval = bar.interval
for bar in bars:
bar.datetime = convert_tz(bar.datetime)
d = {
"measurement": "bar_data",
"tags": {
"vt_symbol": vt_symbol,
"interval": interval.value
},
"time": bar.datetime.isoformat(),
"fields": {
"open_price": bar.open_price,
"high_price": bar.high_price,
"low_price": bar.low_price,
"close_price": bar.close_price,
"volume": bar.volume,
"open_interest": bar.open_interest,
}
}
json_body.append(d)
self.client.write_points(json_body, batch_size=10000)
# Update bar overview
symbol, exchange = extract_vt_symbol(vt_symbol)
key = f"{vt_symbol}_{interval.value}"
f = shelve.open(self.overview_filepath)
overview = f.get(key, None)
if not overview:
overview = BarOverview(
symbol=symbol,
exchange=exchange,
interval=interval
)
overview.count = len(bars)
overview.start = bars[0].datetime
overview.end = bars[-1].datetime
else:
overview.start = min(overview.start, bars[0].datetime)
overview.end = max(overview.end, bars[-1].datetime)
query = (
"select count(close_price) from bar_data"
" where vt_symbol=$vt_symbol"
" and interval=$interval"
)
bind_params = {
"vt_symbol": vt_symbol,
"interval": interval.value
}
result = self.client.query(query, bind_params=bind_params)
points = result.get_points()
for d in points:
overview.count = d["count"]
f[key] = overview
f.close()
