def test_text_reader(self):
provider = BetfairInstrumentProvider.from_instruments([])
reader = BetfairTestStubs.betfair_reader(provider) # type: TextReader
raw_file = make_raw_files(
glob_path=f"{TEST_DATA_DIR}/betfair/1.166811431.bz2")[0]
result = process_raw_file(catalog=self.catalog,
raw_file=raw_file,
reader=reader)
assert result == 22692
def test_text_reader_instrument(self):
def parser(line):
from decimal import Decimal
from nautilus_trader.model.currencies import BTC
from nautilus_trader.model.currencies import USDT
from nautilus_trader.model.enums import AssetClass
from nautilus_trader.model.enums import AssetType
from nautilus_trader.model.identifiers import InstrumentId
from nautilus_trader.model.identifiers import Symbol
from nautilus_trader.model.identifiers import Venue
from nautilus_trader.model.objects import Price
from nautilus_trader.model.objects import Quantity
assert ( # type: ignore # noqa: F631
Decimal,
AssetType,
AssetClass,
USDT,
BTC,
CurrencyPair,
InstrumentId,
Symbol,
Venue,
Price,
Quantity,
) # Ensure imports stay
# Replace str repr with "fully qualified" string we can `eval`
replacements = {
b"id=BTCUSDT.BINANCE":
b"instrument_id=InstrumentId(Symbol('BTCUSDT'), venue=Venue('BINANCE'))",
b"native_symbol=BTCUSDT": b"native_symbol=Symbol('BTCUSDT')",
b"price_increment=0.01":
b"price_increment=Price.from_str('0.01')",
b"size_increment=0.000001":
b"size_increment=Quantity.from_str('0.000001')",
b"margin_init=0": b"margin_init=Decimal(0)",
b"margin_maint=0": b"margin_maint=Decimal(0)",
b"maker_fee=0.001": b"maker_fee=Decimal(0.001)",
b"taker_fee=0.001": b"taker_fee=Decimal(0.001)",
}
for k, v in replacements.items():
line = line.replace(k, v)
yield eval(line) # noqa: S307
reader = TextReader(line_parser=parser)
raw_file = make_raw_files(
glob_path=f"{TEST_DATA_DIR}/binance-btcusdt-instrument.txt")[0]
result = process_raw_file(catalog=self.catalog,
raw_file=raw_file,
reader=reader)
expected = 1
assert result == expected
def test_byte_json_parser(self):
def parser(block):
for data in orjson.loads(block):
obj = CurrencyPair.from_dict(data)
yield obj
reader = ByteReader(block_parser=parser)
raw_file = make_raw_files(glob_path=f"{TEST_DATA_DIR}/crypto*.json")[0]
result = process_raw_file(catalog=self.catalog,
raw_file=raw_file,
reader=reader)
assert result == 6
def betfair_feed_parsed(market_id="1.166564490", folder="data"):
instrument_provider = BetfairInstrumentProvider.from_instruments([])
reader = BetfairTestStubs.betfair_reader(
instrument_provider=instrument_provider)
files = make_raw_files(
glob_path=f"{PACKAGE_ROOT}/{folder}/{market_id}*")
data = []
for rf in files:
for block in rf.iter():
data.extend(reader.parse(block=block))
return data
def test_csv_reader_dataframe(self):
def parser(data):
if data is None:
return
data.loc[:, "timestamp"] = pd.to_datetime(data["timestamp"])
instrument = TestInstrumentProvider.default_fx_ccy("AUD/USD")
wrangler = QuoteTickDataWrangler(instrument)
ticks = wrangler.process(data.set_index("timestamp"))
yield from ticks
reader = CSVReader(block_parser=parser, as_dataframe=True)
raw_file = make_raw_files(
glob_path=f"{TEST_DATA_DIR}/truefx-audusd-ticks.csv")[0]
result = process_raw_file(catalog=self.catalog,
raw_file=raw_file,
reader=reader)
assert result == 100000
def test_parquet_reader(self):
def parser(data):
if data is None:
return
data.loc[:, "timestamp"] = pd.to_datetime(data["timestamp"])
data = data.set_index("timestamp")[[
"bid", "ask", "bid_size", "ask_size"
]]
instrument = TestInstrumentProvider.default_fx_ccy("AUD/USD")
wrangler = QuoteTickDataWrangler(instrument)
ticks = wrangler.process(data)
yield from ticks
reader = ParquetReader(parser=parser)
raw_file = make_raw_files(
glob_path=f"{TEST_DATA_DIR}/binance-btcusdt-quotes.parquet")[0]
result = process_raw_file(catalog=self.catalog,
raw_file=raw_file,
reader=reader)
assert result == 451