def setup_news_event_persistence():
import pyarrow as pa
def _news_event_to_dict(self):
return {
"name": self.name,
"impact": self.impact.name,
"currency": self.currency.code,
"ts_event": self.ts_event,
"ts_init": self.ts_init,
}
def _news_event_from_dict(data):
data.update(
{
"impact": getattr(NewsImpact, data["impact"]),
"currency": Currency.from_str(data["currency"]),
}
)
return NewsEventData(**data)
register_parquet(
cls=NewsEventData,
serializer=_news_event_to_dict,
deserializer=_news_event_from_dict,
partition_keys=("currency",),
schema=pa.schema(
{
"name": pa.string(),
"impact": pa.string(),
"currency": pa.string(),
"ts_event": pa.int64(),
"ts_init": pa.int64(),
}
),
force=True,
)
"multiplier": pa.dictionary(pa.int8(), pa.string()),
"lot_size": pa.dictionary(pa.int8(), pa.string()),
"expiry_date": pa.dictionary(pa.int8(), pa.string()),
"ts_init": pa.int64(),
"ts_event": pa.int64(),
}),
Option:
pa.schema({
"id": pa.dictionary(pa.int64(), pa.string()),
"local_symbol": pa.string(),
"underlying": pa.dictionary(pa.int8(), pa.string()),
"asset_class": pa.dictionary(pa.int8(), pa.string()),
"currency": pa.dictionary(pa.int8(), pa.string()),
"price_precision": pa.int64(),
"size_precision": pa.int64(),
"price_increment": pa.dictionary(pa.int8(), pa.string()),
"size_increment": pa.dictionary(pa.int8(), pa.string()),
"multiplier": pa.dictionary(pa.int8(), pa.string()),
"lot_size": pa.dictionary(pa.int8(), pa.string()),
"expiry_date": pa.dictionary(pa.int64(), pa.string()),
"strike_price": pa.dictionary(pa.int64(), pa.string()),
"kind": pa.dictionary(pa.int8(), pa.string()),
"ts_init": pa.int64(),
"ts_event": pa.int64(),
}),
}
# default schemas
for cls, schema in NAUTILUS_PARQUET_SCHEMA.items():
register_parquet(cls, schema=schema)
# -------------------------------------------------------------------------------------------------
# Copyright (C) 2015-2021 Nautech Systems Pty Ltd. All rights reserved.
# https://nautechsystems.io
#
# Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# -------------------------------------------------------------------------------------------------
from nautilus_trader.model.data.venue import InstrumentClosePrice
from nautilus_trader.serialization.arrow.serializer import register_parquet
def serialize(price: InstrumentClosePrice):
result = price.to_dict(price)
result["close_price"] = price.close_price.as_double()
return result
register_parquet(InstrumentClosePrice, serializer=serialize)
instrument_id=v["margin_instrument_id"],
))
state = {
k: v
for k, v in values[0].items()
if not k.startswith("balance_") and not k.startswith("margin_")
}
state["balances"] = orjson.dumps(balances)
state["margins"] = orjson.dumps(margins)
return AccountState.from_dict(state)
def deserialize(data: List[Dict]):
results = []
for _, chunk in itertools.groupby(sorted(data,
key=lambda x: x["event_id"]),
key=lambda x: x["event_id"]):
chunk = list(chunk) # type: ignore
results.append(_deserialize(values=chunk))
return sorted(results, key=lambda x: x.ts_init)
register_parquet(
AccountState,
serializer=serialize,
deserializer=deserialize,
chunk=True,
)
ticker.ts_init,
"last_traded_price":
str(ticker.last_traded_price) if ticker.last_traded_price else None,
"traded_volume":
str(ticker.traded_volume) if ticker.traded_volume else None,
}
BSP_SCHEMA = pa.schema(
{
"instrument_id": pa.string(),
"ts_event": pa.int64(),
"ts_init": pa.int64(),
"action": pa.string(),
"order_side": pa.string(),
"order_price": pa.float64(),
"order_size": pa.float64(),
"order_id": pa.string(),
"book_type": pa.string(),
},
metadata={"type": "BSPOrderBookDelta"},
)
register_serializable_object(BetfairTicker, betfair_ticker_to_dict,
betfair_ticker_from_dict)
register_parquet(cls=BetfairTicker, schema=BetfairTicker.schema())
register_serializable_object(BSPOrderBookDelta, BSPOrderBookDelta.to_dict,
BSPOrderBookDelta.from_dict)
register_parquet(cls=BSPOrderBookDelta, schema=BSP_SCHEMA)
for k, v in data.items()
}
return data
def deserialize_order_filled(data: Dict) -> OrderFilled:
for k in ("last_px", "last_qty"):
data[k] = str(data[k])
return OrderFilled.from_dict(data)
def deserialize_order_initialised(data: Dict) -> OrderInitialized:
for k in ("price", "quantity"):
data[k] = str(data[k])
options_fields = orjson.loads(
NAUTILUS_PARQUET_SCHEMA[OrderInitialized].metadata[b"options_fields"])
data["options"] = orjson.dumps(
{k: data.pop(k, None)
for k in options_fields})
return OrderInitialized.from_dict(data)
register_parquet(OrderFilled,
serializer=serialize,
deserializer=deserialize_order_filled)
register_parquet(
OrderInitialized,
serializer=serialize_order_initialized,
deserializer=deserialize_order_initialised,
)
values["realized_pnl"] = realized.as_double()
if "unrealized_pnl" in values:
unrealized = Money.from_str(values["unrealized_pnl"])
values["unrealized_pnl"] = unrealized.as_double()
return values
def deserialize(cls):
def inner(
data: Dict
) -> Union[PositionOpened, PositionChanged, PositionClosed]:
for k in ("net_qty", "quantity", "last_qty", "peak_qty", "last_px"):
if k in data:
data[k] = str(data[k])
if "realized_pnl" in data:
data["realized_pnl"] = f"{data['realized_pnl']} {data['currency']}"
if "unrealized_pnl" in data:
data[
"unrealized_pnl"] = f"{data['unrealized_pnl']} {data['currency']}"
return cls.from_dict(data)
return inner
for cls in (PositionOpened, PositionChanged, PositionClosed):
register_parquet(
cls,
serializer=serialize,
deserializer=deserialize(cls=cls),
)
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# -------------------------------------------------------------------------------------------------
from typing import Dict
from nautilus_trader.model.data.bar import Bar
from nautilus_trader.serialization.arrow.serializer import register_parquet
def serialize(bar: Bar):
data = bar.to_dict(bar)
data["instrument_id"] = bar.type.instrument_id.value
return data
def deserialize(data: Dict) -> Bar:
ignore = ("instrument_id", )
bar = Bar.from_dict({k: v for k, v in data.items() if k not in ignore})
return bar
register_parquet(
Bar,
serializer=serialize,
deserializer=deserialize,
)
ts_init=values[0]["ts_init"],
)
def _sort_func(x):
return x["instrument_id"], x["ts_event"]
def deserialize(data: List[Dict]):
assert not set([d["order_side"] for d in data]).difference(
(None, "BUY", "SELL")), "Wrong sides"
results = []
for _, chunk in itertools.groupby(sorted(data, key=_sort_func),
key=_sort_func):
chunk = list(chunk) # type: ignore
if _is_orderbook_snapshot(values=chunk): # type: ignore
results.append(_build_order_book_snapshot(values=chunk))
elif len(chunk) >= 1: # type: ignore
results.append(_build_order_book_deltas(values=chunk))
return sorted(results, key=lambda x: x.ts_event)
for cls in [OrderBookData] + OrderBookData.__subclasses__():
register_parquet(
cls=cls,
serializer=serialize,
deserializer=deserialize,
table=OrderBookData,
chunk=True,
)
# -------------------------------------------------------------------------------------------------
# Copyright (C) 2015-2021 Nautech Systems Pty Ltd. All rights reserved.
# https://nautechsystems.io
#
# Licensed under the GNU Lesser General Public License Version 3.0 (the "License");
# You may not use this file except in compliance with the License.
# You may obtain a copy of the License at https://www.gnu.org/licenses/lgpl-3.0.en.html
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# -------------------------------------------------------------------------------------------------
from nautilus_trader.model.instruments.base import Instrument
from nautilus_trader.serialization.arrow.serializer import register_parquet
for cls in Instrument.__subclasses__():
register_parquet(cls, partition_keys=tuple())
