def load_bar_data( self, symbol: str, exchange: Exchange, interval: Interval, start: datetime, end: datetime ) -> List[BarData]: """""" s: QuerySet = DbBarData.objects( symbol=symbol, exchange=exchange.value, interval=interval.value, datetime__gte=convert_tz(start), datetime__lte=convert_tz(end), ) vt_symbol = f"{symbol}.{exchange.value}" bars: List[BarData] = [] for db_bar in s: db_bar.datetime = DB_TZ.localize(db_bar.datetime) db_bar.exchange = Exchange(db_bar.exchange) db_bar.interval = Interval(db_bar.interval) db_bar.gateway_name = "DB" db_bar.vt_symbol = vt_symbol bars.append(db_bar) return bars
def load_tick_data( self, symbol: str, exchange: Exchange, start: datetime, end: datetime ) -> List[TickData]: """""" s: QuerySet = DbTickData.objects( symbol=symbol, exchange=exchange.value, datetime__gte=convert_tz(start), datetime__lte=convert_tz(end), ) vt_symbol = f"{symbol}.{exchange.value}" ticks: List[TickData] = [] for db_tick in s: db_tick.datetime = DB_TZ.localize(db_tick.datetime) db_tick.exchange = Exchange(db_tick.exchange) db_tick.gateway_name = "DB" db_tick.vt_symbol = vt_symbol ticks.append(db_tick) return ticks
def load_tick_data( self, symbol: str, exchange: Exchange, start: datetime, end: datetime ) -> List[TickData]: """""" s: QuerySet = DbTickData.objects( symbol=symbol, exchange=exchange.value, datetime__gte=convert_tz(start), datetime__lte=convert_tz(end), ) ticks: List[TickData] = [] for db_tick in s: tick = TickData( symbol=db_tick.symbol, exchange=Exchange(db_tick.exchange), datetime=db_tick.datetime.astimezone(DB_TZ), name=db_tick.name, volume=db_tick.volume, turnover=db_tick.turnover, open_interest=db_tick.open_interest, last_price=db_tick.last_price, last_volume=db_tick.last_volume, limit_up=db_tick.limit_up, limit_down=db_tick.limit_down, open_price=db_tick.open_price, high_price=db_tick.high_price, low_price=db_tick.low_price, pre_close=db_tick.pre_close, bid_price_1=db_tick.bid_price_1, bid_price_2=db_tick.bid_price_2, bid_price_3=db_tick.bid_price_3, bid_price_4=db_tick.bid_price_4, bid_price_5=db_tick.bid_price_5, ask_price_1=db_tick.ask_price_1, ask_price_2=db_tick.ask_price_2, ask_price_3=db_tick.ask_price_3, ask_price_4=db_tick.ask_price_4, ask_price_5=db_tick.ask_price_5, bid_volume_1=db_tick.bid_volume_1, bid_volume_2=db_tick.bid_volume_2, bid_volume_3=db_tick.bid_volume_3, bid_volume_4=db_tick.bid_volume_4, bid_volume_5=db_tick.bid_volume_5, ask_volume_1=db_tick.ask_volume_1, ask_volume_2=db_tick.ask_volume_2, ask_volume_3=db_tick.ask_volume_3, ask_volume_4=db_tick.ask_volume_4, ask_volume_5=db_tick.ask_volume_5, localtime=db_tick.localtime, gateway_name="DB" ) ticks.append(tick) return ticks
def save_tick_data(self, ticks: List[TickData]) -> bool: """""" # Convert bar object to dict and adjust timezone data = [] for tick in ticks: tick.datetime = convert_tz(tick.datetime) d = tick.__dict__ d["exchange"] = d["exchange"].value d.pop("gateway_name") d.pop("vt_symbol") data.append(d) # Upsert data into database with self.db.atomic(): for d in data: DbTickData.insert(d).on_conflict( update=d, conflict_target=( DbTickData.symbol, DbTickData.exchange, DbTickData.datetime, ), ).execute()
def save_bar_data(self, bars: List[BarData]) -> bool: """""" # Store key parameters bar = bars[0] symbol = bar.symbol exchange = bar.exchange interval = bar.interval # Convert bar object to dict and adjust timezone data = [] for bar in bars: bar.datetime = convert_tz(bar.datetime) d = bar.__dict__ d["exchange"] = d["exchange"].value d["interval"] = d["interval"].value d.pop("gateway_name") d.pop("vt_symbol") data.append(d) # Upsert data into database with self.db.atomic(): for d in data: DbBarData.insert(d).on_conflict( update=d, conflict_target=( DbBarData.symbol, DbBarData.exchange, DbBarData.interval, DbBarData.datetime, ), ).execute() # Update bar overview overview: DbBarOverview = DbBarOverview.get_or_none( DbBarOverview.symbol == symbol, DbBarOverview.exchange == exchange.value, DbBarOverview.interval == interval.value, ) if not overview: overview = DbBarOverview() overview.symbol = symbol overview.exchange = exchange.value overview.interval = interval.value overview.start = bars[0].datetime overview.end = bars[-1].datetime overview.count = len(bars) else: overview.start = min(bars[0].datetime, overview.start) overview.end = max(bars[-1].datetime, overview.end) s: ModelSelect = DbBarData.select().where( (DbBarData.symbol == symbol) & (DbBarData.exchange == exchange.value) & (DbBarData.interval == interval.value)) overview.count = s.count() overview.save()
def save_tick_data(self, ticks: List[TickData], collection_name: str = None) -> bool: """""" for tick in ticks: tick.datetime = convert_tz(tick.datetime) d = tick.__dict__ d["exchange"] = d["exchange"].value d["interval"] = d["interval"].value d.pop("gateway_name") d.pop("vt_symbol") param = to_update_param(d) if not collection_name: DbTickData.objects( symbol=d["symbol"], exchange=d["exchange"], datetime=d["datetime"], ).update_one(upsert=True, **param) else: with switch_collection(DbTickData, collection_name): DbTickData.objects( symbol=d["symbol"], exchange=d["exchange"], datetime=d["datetime"], ).update_one(upsert=True, **param)
def save_tick_data(self, ticks: List[TickData]) -> bool: """""" json_body = [] tick = ticks[0] vt_symbol = tick.vt_symbol for tick in ticks: tick.datetime = convert_tz(tick.datetime) d = { "measurement": "tick_data", "tags": { "vt_symbol": vt_symbol }, "time": tick.datetime.isoformat(), "fields": { "name": tick.name, "volume": tick.volume, "open_interest": tick.open_interest, "last_price": tick.last_price, "last_volume": tick.last_volume, "limit_up": tick.limit_up, "limit_down": tick.limit_down, "open_price": tick.open_price, "high_price": tick.high_price, "low_price": tick.low_price, "pre_close": tick.pre_close, "bid_price_1": tick.bid_price_1, "bid_price_2": tick.bid_price_2, "bid_price_3": tick.bid_price_3, "bid_price_4": tick.bid_price_4, "bid_price_5": tick.bid_price_5, "ask_price_1": tick.ask_price_1, "ask_price_2": tick.ask_price_2, "ask_price_3": tick.ask_price_3, "ask_price_4": tick.ask_price_4, "ask_price_5": tick.ask_price_5, "bid_volume_1": tick.bid_volume_1, "bid_volume_2": tick.bid_volume_2, "bid_volume_3": tick.bid_volume_3, "bid_volume_4": tick.bid_volume_4, "bid_volume_5": tick.bid_volume_5, "ask_volume_1": tick.ask_volume_1, "ask_volume_2": tick.ask_volume_2, "ask_volume_3": tick.ask_volume_3, "ask_volume_4": tick.ask_volume_4, "ask_volume_5": tick.ask_volume_5, } } json_body.append(d) self.client.write_points(json_body, batch_size=10000)
def save_bar_data(self, bars: List[BarData]) -> bool: """保存K线数据""" # Store key parameters bar = bars[0] symbol = bar.symbol exchange = bar.exchange interval = bar.interval # Convert bar object to dict and adjust timezone data = [] for bar in bars: bar.datetime = convert_tz(bar.datetime) data = DbBarData( symbol=bar.symbol, exchange=exchange.value, datetime=bar.datetime, interval=interval.value, volume=bar.volume, open_interest=bar.open_interest, open_price=bar.open_price, high_price=bar.high_price, low_price=bar.low_price, close_price=bar.close_price ) self.db.merge(data) self.db.commit() overview = self.db.query(DbBarOverview).filter( DbBarOverview.symbol == symbol, DbBarOverview.exchange == exchange.value, DbBarOverview.interval == interval.value ).first() if not overview: overview = DbBarOverview( symbol=symbol, exchange=exchange.value, interval=interval.value, count=len(bars), start=bars[0].datetime, end=bars[-1].datetime ) else: overview.start = min(bars[0].datetime, overview.start) overview.end = max(bars[-1].datetime, overview.end) overview.count = self.db.query(DbBarData).filter( DbBarData.symbol == symbol, DbBarData.exchange == exchange.value, DbBarData.interval == interval.value ).count() self.db.merge(overview) self.db.commit()
def save_bar_data(self, bars: List[BarData]) -> bool: """""" # Store key parameters bar = bars[0] symbol = bar.symbol exchange = bar.exchange interval = bar.interval # Upsert data into mongodb for bar in bars: bar.datetime = convert_tz(bar.datetime) d = bar.__dict__ d["exchange"] = d["exchange"].value d["interval"] = d["interval"].value d.pop("gateway_name") d.pop("vt_symbol") param = to_update_param(d) DbBarData.objects( symbol=d["symbol"], exchange=d["exchange"], interval=d["interval"], datetime=d["datetime"], ).update_one(upsert=True, **param) # Update bar overview try: overview: DbBarOverview = DbBarOverview.objects( symbol=symbol, exchange=exchange.value, interval=interval.value ).get() except DoesNotExist: overview: DbBarOverview = DbBarOverview( symbol=symbol, exchange=exchange.value, interval=interval.value ) if not overview.start: overview.start = bars[0].datetime overview.end = bars[-1].datetime overview.count = len(bars) else: overview.start = min(bars[0].datetime, overview.start) overview.end = max(bars[-1].datetime, overview.end) overview.count = DbBarData.objects( symbol=symbol, exchange=exchange.value, interval=interval.value ).count() overview.save()
def load_bar_data( self, symbol: str, exchange: Exchange, interval: Interval, start: datetime, end: datetime ) -> List[BarData]: """""" s: QuerySet = DbBarData.objects( symbol=symbol, exchange=exchange.value, interval=interval.value, datetime__gte=convert_tz(start), datetime__lte=convert_tz(end), ) bars: List[BarData] = [] for db_bar in s: bar = BarData( symbol=db_bar.symbol, exchange=Exchange(db_bar.exchange), datetime=db_bar.datetime.astimezone(DB_TZ), interval=Interval(db_bar.interval), volume=db_bar.volume, turnover=db_bar.turnover, open_interest=db_bar.open_interest, open_price=db_bar.open_price, high_price=db_bar.high_price, low_price=db_bar.low_price, close_price=db_bar.close_price, gateway_name="DB" ) bars.append(bar) return bars
def save_tick_data(self, ticks: List[TickData]) -> bool: """""" # Convert tick object to dict and adjust timezone tick = ticks[0] exchange = tick.exchange data = [] for tick in ticks: tick.datetime = convert_tz(tick.datetime) data = DbTickData( symbol=tick.symbol, exchange=exchange.value, datetime=tick.datetime, volume=tick.volume, open_interest=tick.open_interest, last_price=tick.last_price, last_volume=tick.last_volume, limit_up=tick.limit_up, limit_down=tick.limit_down, open_price=tick.open_price, high_price=tick.high_price, low_price=tick.low_price, pre_close=tick.pre_close, bid_price_1=tick.bid_price_1, bid_price_2=tick.bid_price_2, bid_price_3=tick.bid_price_3, bid_price_4=tick.bid_price_4, bid_price_5=tick.bid_price_5, ask_price_1=tick.ask_price_1, ask_price_2=tick.ask_price_2, ask_price_3=tick.ask_price_3, ask_price_4=tick.ask_price_4, ask_price_5=tick.ask_price_5, bid_volume_1=tick.bid_volume_1, bid_volume_2=tick.bid_volume_2, bid_volume_3=tick.bid_volume_3, bid_volume_4=tick.bid_volume_4, bid_volume_5=tick.bid_volume_5, ask_volume_1=tick.ask_volume_1, ask_volume_2=tick.ask_volume_2, ask_volume_3=tick.ask_volume_3, ask_volume_4=tick.ask_volume_4, ask_volume_5=tick.ask_volume_5 ) self.db.merge(data) self.db.commit()
def save_tick_data(self, ticks: List[TickData]) -> bool: """""" for tick in ticks: tick.datetime = convert_tz(tick.datetime) d = tick.__dict__ d["exchange"] = d["exchange"].value d.pop("gateway_name") d.pop("vt_symbol") param = to_update_param(d) DbTickData.objects( symbol=d["symbol"], exchange=d["exchange"], datetime=d["datetime"], ).update_one(upsert=True, **param)
def save_tick_data(self, ticks: List[TickData]) -> bool: """""" # Convert bar object to dict and adjust timezone data = [] for tick in ticks: tick.datetime = convert_tz(tick.datetime) d = tick.__dict__ d["exchange"] = d["exchange"].value d.pop("gateway_name") d.pop("vt_symbol") data.append(d) # Upsert data into database with self.db.atomic(): for c in chunked(data, 50): DbTickData.insert_many(c).on_conflict_replace().execute()
def save_bar_data(self, bars: List[BarData]) -> bool: """""" json_body = [] bar = bars[0] vt_symbol = bar.vt_symbol interval = bar.interval for bar in bars: bar.datetime = convert_tz(bar.datetime) d = { "measurement": "bar_data", "tags": { "vt_symbol": vt_symbol, "interval": interval.value }, "time": bar.datetime.isoformat(), "fields": { "open_price": bar.open_price, "high_price": bar.high_price, "low_price": bar.low_price, "close_price": bar.close_price, "volume": bar.volume, "open_interest": bar.open_interest, } } json_body.append(d) self.client.write_points(json_body, batch_size=10000) # Update bar overview symbol, exchange = extract_vt_symbol(vt_symbol) key = f"{vt_symbol}_{interval.value}" f = shelve.open(self.overview_filepath) overview = f.get(key, None) if not overview: overview = BarOverview( symbol=symbol, exchange=exchange, interval=interval ) overview.count = len(bars) overview.start = bars[0].datetime overview.end = bars[-1].datetime else: overview.start = min(overview.start, bars[0].datetime) overview.end = max(overview.end, bars[-1].datetime) query = ( "select count(close_price) from bar_data" " where vt_symbol=$vt_symbol" " and interval=$interval" ) bind_params = { "vt_symbol": vt_symbol, "interval": interval.value } result = self.client.query(query, bind_params=bind_params) points = result.get_points() for d in points: overview.count = d["count"] f[key] = overview f.close()