def setup_news_event_persistence(): import pyarrow as pa def _news_event_to_dict(self): return { "name": self.name, "impact": self.impact.name, "currency": self.currency.code, "ts_event": self.ts_event, "ts_init": self.ts_init, } def _news_event_from_dict(data): data.update( { "impact": getattr(NewsImpact, data["impact"]), "currency": Currency.from_str(data["currency"]), } ) return NewsEventData(**data) register_parquet( cls=NewsEventData, serializer=_news_event_to_dict, deserializer=_news_event_from_dict, partition_keys=("currency",), schema=pa.schema( { "name": pa.string(), "impact": pa.string(), "currency": pa.string(), "ts_event": pa.int64(), "ts_init": pa.int64(), } ), force=True, )
"multiplier": pa.dictionary(pa.int8(), pa.string()), "lot_size": pa.dictionary(pa.int8(), pa.string()), "expiry_date": pa.dictionary(pa.int8(), pa.string()), "ts_init": pa.int64(), "ts_event": pa.int64(), }), Option: pa.schema({ "id": pa.dictionary(pa.int64(), pa.string()), "local_symbol": pa.string(), "underlying": pa.dictionary(pa.int8(), pa.string()), "asset_class": pa.dictionary(pa.int8(), pa.string()), "currency": pa.dictionary(pa.int8(), pa.string()), "price_precision": pa.int64(), "size_precision": pa.int64(), "price_increment": pa.dictionary(pa.int8(), pa.string()), "size_increment": pa.dictionary(pa.int8(), pa.string()), "multiplier": pa.dictionary(pa.int8(), pa.string()), "lot_size": pa.dictionary(pa.int8(), pa.string()), "expiry_date": pa.dictionary(pa.int64(), pa.string()), "strike_price": pa.dictionary(pa.int64(), pa.string()), "kind": pa.dictionary(pa.int8(), pa.string()), "ts_init": pa.int64(), "ts_event": pa.int64(), }), } # default schemas for cls, schema in NAUTILUS_PARQUET_SCHEMA.items(): register_parquet(cls, schema=schema)
# ------------------------------------------------------------------------------------------------- # Copyright (C) 2015-2021 Nautech Systems Pty Ltd. All rights reserved. # https://nautechsystems.io # # Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); # You may not use this file except in compliance with the License. # You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ------------------------------------------------------------------------------------------------- from nautilus_trader.model.data.venue import InstrumentClosePrice from nautilus_trader.serialization.arrow.serializer import register_parquet def serialize(price: InstrumentClosePrice): result = price.to_dict(price) result["close_price"] = price.close_price.as_double() return result register_parquet(InstrumentClosePrice, serializer=serialize)
instrument_id=v["margin_instrument_id"], )) state = { k: v for k, v in values[0].items() if not k.startswith("balance_") and not k.startswith("margin_") } state["balances"] = orjson.dumps(balances) state["margins"] = orjson.dumps(margins) return AccountState.from_dict(state) def deserialize(data: List[Dict]): results = [] for _, chunk in itertools.groupby(sorted(data, key=lambda x: x["event_id"]), key=lambda x: x["event_id"]): chunk = list(chunk) # type: ignore results.append(_deserialize(values=chunk)) return sorted(results, key=lambda x: x.ts_init) register_parquet( AccountState, serializer=serialize, deserializer=deserialize, chunk=True, )
ticker.ts_init, "last_traded_price": str(ticker.last_traded_price) if ticker.last_traded_price else None, "traded_volume": str(ticker.traded_volume) if ticker.traded_volume else None, } BSP_SCHEMA = pa.schema( { "instrument_id": pa.string(), "ts_event": pa.int64(), "ts_init": pa.int64(), "action": pa.string(), "order_side": pa.string(), "order_price": pa.float64(), "order_size": pa.float64(), "order_id": pa.string(), "book_type": pa.string(), }, metadata={"type": "BSPOrderBookDelta"}, ) register_serializable_object(BetfairTicker, betfair_ticker_to_dict, betfair_ticker_from_dict) register_parquet(cls=BetfairTicker, schema=BetfairTicker.schema()) register_serializable_object(BSPOrderBookDelta, BSPOrderBookDelta.to_dict, BSPOrderBookDelta.from_dict) register_parquet(cls=BSPOrderBookDelta, schema=BSP_SCHEMA)
for k, v in data.items() } return data def deserialize_order_filled(data: Dict) -> OrderFilled: for k in ("last_px", "last_qty"): data[k] = str(data[k]) return OrderFilled.from_dict(data) def deserialize_order_initialised(data: Dict) -> OrderInitialized: for k in ("price", "quantity"): data[k] = str(data[k]) options_fields = orjson.loads( NAUTILUS_PARQUET_SCHEMA[OrderInitialized].metadata[b"options_fields"]) data["options"] = orjson.dumps( {k: data.pop(k, None) for k in options_fields}) return OrderInitialized.from_dict(data) register_parquet(OrderFilled, serializer=serialize, deserializer=deserialize_order_filled) register_parquet( OrderInitialized, serializer=serialize_order_initialized, deserializer=deserialize_order_initialised, )
values["realized_pnl"] = realized.as_double() if "unrealized_pnl" in values: unrealized = Money.from_str(values["unrealized_pnl"]) values["unrealized_pnl"] = unrealized.as_double() return values def deserialize(cls): def inner( data: Dict ) -> Union[PositionOpened, PositionChanged, PositionClosed]: for k in ("net_qty", "quantity", "last_qty", "peak_qty", "last_px"): if k in data: data[k] = str(data[k]) if "realized_pnl" in data: data["realized_pnl"] = f"{data['realized_pnl']} {data['currency']}" if "unrealized_pnl" in data: data[ "unrealized_pnl"] = f"{data['unrealized_pnl']} {data['currency']}" return cls.from_dict(data) return inner for cls in (PositionOpened, PositionChanged, PositionClosed): register_parquet( cls, serializer=serialize, deserializer=deserialize(cls=cls), )
# Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ------------------------------------------------------------------------------------------------- from typing import Dict from nautilus_trader.model.data.bar import Bar from nautilus_trader.serialization.arrow.serializer import register_parquet def serialize(bar: Bar): data = bar.to_dict(bar) data["instrument_id"] = bar.type.instrument_id.value return data def deserialize(data: Dict) -> Bar: ignore = ("instrument_id", ) bar = Bar.from_dict({k: v for k, v in data.items() if k not in ignore}) return bar register_parquet( Bar, serializer=serialize, deserializer=deserialize, )
ts_init=values[0]["ts_init"], ) def _sort_func(x): return x["instrument_id"], x["ts_event"] def deserialize(data: List[Dict]): assert not set([d["order_side"] for d in data]).difference( (None, "BUY", "SELL")), "Wrong sides" results = [] for _, chunk in itertools.groupby(sorted(data, key=_sort_func), key=_sort_func): chunk = list(chunk) # type: ignore if _is_orderbook_snapshot(values=chunk): # type: ignore results.append(_build_order_book_snapshot(values=chunk)) elif len(chunk) >= 1: # type: ignore results.append(_build_order_book_deltas(values=chunk)) return sorted(results, key=lambda x: x.ts_event) for cls in [OrderBookData] + OrderBookData.__subclasses__(): register_parquet( cls=cls, serializer=serialize, deserializer=deserialize, table=OrderBookData, chunk=True, )
# ------------------------------------------------------------------------------------------------- # Copyright (C) 2015-2021 Nautech Systems Pty Ltd. All rights reserved. # https://nautechsystems.io # # Licensed under the GNU Lesser General Public License Version 3.0 (the "License"); # You may not use this file except in compliance with the License. # You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # ------------------------------------------------------------------------------------------------- from nautilus_trader.model.instruments.base import Instrument from nautilus_trader.serialization.arrow.serializer import register_parquet for cls in Instrument.__subclasses__(): register_parquet(cls, partition_keys=tuple())