def test_bsp_deltas(self): rf = RawFile(open_file=fsspec.open( f"{PACKAGE_ROOT}/data/betfair/1.170258150.bz2", compression="infer")) process_raw_file(catalog=self.catalog, raw_file=rf, reader=self.reader) data = self.catalog.query(BSPOrderBookDelta) assert len(data) == 443
def test_raw_file_process(self): # Arrange rf = RawFile( open_file=fsspec.open(f"{TEST_DATA}/1.166564490.bz2", compression="infer"), block_size=None, ) # Act process_raw_file(catalog=self.catalog, reader=make_betfair_reader(), raw_file=rf) # Assert assert len(self.catalog.instruments()) == 2
def test_text_reader(self): provider = BetfairInstrumentProvider.from_instruments([]) reader = BetfairTestStubs.betfair_reader(provider) # type: TextReader raw_file = make_raw_files( glob_path=f"{TEST_DATA_DIR}/betfair/1.166811431.bz2")[0] result = process_raw_file(catalog=self.catalog, raw_file=raw_file, reader=reader) assert result == 22692
def test_text_reader_instrument(self): def parser(line): from decimal import Decimal from nautilus_trader.model.currencies import BTC from nautilus_trader.model.currencies import USDT from nautilus_trader.model.enums import AssetClass from nautilus_trader.model.enums import AssetType from nautilus_trader.model.identifiers import InstrumentId from nautilus_trader.model.identifiers import Symbol from nautilus_trader.model.identifiers import Venue from nautilus_trader.model.objects import Price from nautilus_trader.model.objects import Quantity assert ( # type: ignore # noqa: F631 Decimal, AssetType, AssetClass, USDT, BTC, CurrencyPair, InstrumentId, Symbol, Venue, Price, Quantity, ) # Ensure imports stay # Replace str repr with "fully qualified" string we can `eval` replacements = { b"id=BTCUSDT.BINANCE": b"instrument_id=InstrumentId(Symbol('BTCUSDT'), venue=Venue('BINANCE'))", b"native_symbol=BTCUSDT": b"native_symbol=Symbol('BTCUSDT')", b"price_increment=0.01": b"price_increment=Price.from_str('0.01')", b"size_increment=0.000001": b"size_increment=Quantity.from_str('0.000001')", b"margin_init=0": b"margin_init=Decimal(0)", b"margin_maint=0": b"margin_maint=Decimal(0)", b"maker_fee=0.001": b"maker_fee=Decimal(0.001)", b"taker_fee=0.001": b"taker_fee=Decimal(0.001)", } for k, v in replacements.items(): line = line.replace(k, v) yield eval(line) # noqa: S307 reader = TextReader(line_parser=parser) raw_file = make_raw_files( glob_path=f"{TEST_DATA_DIR}/binance-btcusdt-instrument.txt")[0] result = process_raw_file(catalog=self.catalog, raw_file=raw_file, reader=reader) expected = 1 assert result == expected
def test_byte_json_parser(self): def parser(block): for data in orjson.loads(block): obj = CurrencyPair.from_dict(data) yield obj reader = ByteReader(block_parser=parser) raw_file = make_raw_files(glob_path=f"{TEST_DATA_DIR}/crypto*.json")[0] result = process_raw_file(catalog=self.catalog, raw_file=raw_file, reader=reader) assert result == 6
def test_csv_reader_dataframe(self): def parser(data): if data is None: return data.loc[:, "timestamp"] = pd.to_datetime(data["timestamp"]) instrument = TestInstrumentProvider.default_fx_ccy("AUD/USD") wrangler = QuoteTickDataWrangler(instrument) ticks = wrangler.process(data.set_index("timestamp")) yield from ticks reader = CSVReader(block_parser=parser, as_dataframe=True) raw_file = make_raw_files( glob_path=f"{TEST_DATA_DIR}/truefx-audusd-ticks.csv")[0] result = process_raw_file(catalog=self.catalog, raw_file=raw_file, reader=reader) assert result == 100000
def test_parquet_reader(self): def parser(data): if data is None: return data.loc[:, "timestamp"] = pd.to_datetime(data["timestamp"]) data = data.set_index("timestamp")[[ "bid", "ask", "bid_size", "ask_size" ]] instrument = TestInstrumentProvider.default_fx_ccy("AUD/USD") wrangler = QuoteTickDataWrangler(instrument) ticks = wrangler.process(data) yield from ticks reader = ParquetReader(parser=parser) raw_file = make_raw_files( glob_path=f"{TEST_DATA_DIR}/binance-btcusdt-quotes.parquet")[0] result = process_raw_file(catalog=self.catalog, raw_file=raw_file, reader=reader) assert result == 451