def test_trade_zip_file_stream() -> None:
d = utc_datetime(2018, 1, 1)
with tempfile.TemporaryDirectory() as tmp:
tar_dir = os.path.join(tmp, d.strftime("%Y%m%d"))
stream = MockTradeZipFileStream(point=DatePoint(date=d,
url=mock_url,
dst_dir=tmp),
stream=stream_trade)
stream.process()
with gzip.open(os.path.join(tar_dir, "ADAZ18.csv.gz")) as f:
content = f.read()
assert content == b"""timestamp,symbol,side,size,price,tickDirection,trdMatchID,grossValue,homeNotional,foreignNotional
2018-12-05D00:00:25.948313000,ADAZ18,Sell,166774,9.54e-06,MinusTick,498350a3-8c03-affd-cf87-e6d8a90b70e5,159102396,166774,1.591024
"""
with gzip.open(os.path.join(tar_dir, "XBTUSD.csv.gz")) as f:
content = f.read()
assert content == b"""timestamp,symbol,side,size,price,tickDirection,trdMatchID,grossValue,homeNotional,foreignNotional
2018-12-05D00:00:25.948313000,XBTUSD,Sell,11,9.54e-06,ZeroMinusTick,1d004a95-44fa-c75c-47cd-d651ecd41c25,10494,11,0.00010494
"""
with gzip.open(os.path.join(tar_dir, "XBTZ18.csv.gz")) as f:
content = f.read()
assert content == b"""timestamp,symbol,side,size,price,tickDirection,trdMatchID,grossValue,homeNotional,foreignNotional
def test_quote_hdf_stream() -> None:
with patch("monkq.exchange.bitmex.data.download.requests") as m:
resp = m.get()
resp.raw = io.BytesIO(stream_quote_gzip)
with tempfile.TemporaryDirectory() as tmp:
point = DatePoint(utc_datetime(2018, 1, 3), 'test_url', tmp)
stream = HDFQuoteStream(point=point)
stream.process()
with pandas.HDFStore(os.path.join(tmp, QUOTE_FILE_NAME),
'r') as store:
XBTz18 = store['XBTZ18']
assert XBTz18.index[0] == utc_datetime(2018, 12, 5, 0, 0, 7,
302458)
assert XBTz18['bidSize'][0] == 256083
assert XBTz18['bidPrice'][0] == 9.54e-06
assert XBTz18['askPrice'][0] == 9.55e-06
assert XBTz18['askSize'][0] == 574060
XBTUSD = store['XBTUSD']
assert XBTUSD.index[0] == utc_datetime(2018, 12, 5, 0, 0, 5,
45590)
assert XBTUSD['bidSize'][0] == 256081
assert XBTUSD['bidPrice'][0] == 9.54e-06
assert XBTUSD['askPrice'][0] == 9.55e-06
assert XBTUSD['askSize'][0] == 544060
ADAZ18 = store['ADAZ18']
assert ADAZ18.index[0] == utc_datetime(2018, 12, 5, 0, 0, 4,
778522)
assert ADAZ18['bidSize'][0] == 256089
assert ADAZ18['bidPrice'][0] == 9.54e-06
assert ADAZ18['askPrice'][0] == 9.55e-06
assert ADAZ18['askSize'][0] == 524060
def test_quote_zip_file_stream() -> None:
d = utc_datetime(2018, 1, 1)
with tempfile.TemporaryDirectory() as tmp:
tar_dir = os.path.join(tmp, d.strftime("%Y%m%d"))
stream = MockQuoteZipFileStream(point=DatePoint(date=d,
url=mock_url,
dst_dir=tmp),
stream=stream_quote)
stream.process()
with gzip.open(os.path.join(tar_dir, "ADAZ18.csv.gz")) as f:
content = f.read()
assert content == b"""timestamp,symbol,bidSize,bidPrice,askPrice,askSize
2018-12-05D00:00:04.778522000,ADAZ18,256089,9.54e-06,9.55e-06,524060
"""
with gzip.open(os.path.join(tar_dir, "XBTUSD.csv.gz")) as f:
content = f.read()
assert content == b"""timestamp,symbol,bidSize,bidPrice,askPrice,askSize
2018-12-05D00:00:05.045590000,XBTUSD,256081,9.54e-06,9.55e-06,544060
"""
with gzip.open(os.path.join(tar_dir, "XBTZ18.csv.gz")) as f:
content = f.read()
assert content == b"""timestamp,symbol,bidSize,bidPrice,askPrice,askSize
def test_raw_stream_request_exception() -> None:
with tempfile.TemporaryDirectory() as tmp:
date = utc_datetime(2018, 1, 1)
outcome = os.path.join(tmp, date.strftime("%Y%m%d") + '.csv.gz')
stream = MockRawStreamRequest(point=DatePoint(date, mock_url, tmp),
stream=stream_b)
stream._stream_requests = _mock_exception_stream # type:ignore
with pytest.raises(DataDownloadError):
stream.process()
assert not os.path.exists(outcome)
def test_symbols_stream_request() -> None:
with tempfile.TemporaryDirectory() as tmp:
stream = MockSymbolsStream(point=DatePoint(utc_datetime(2018, 1, 1),
mock_url, tmp),
stream=stream_symbols)
stream.process()
with open(os.path.join(tmp, INSTRUMENT_FILENAME), 'rb') as f:
content = f.read()
assert content == stream_symbols
def test_trade_hdf_stream() -> None:
with patch("monkq.exchange.bitmex.data.download.requests") as m:
resp = m.get()
resp.raw = io.BytesIO(stream_trade_gzip)
with tempfile.TemporaryDirectory() as tmp:
point = DatePoint(
utc_datetime(2018, 1, 3),
'test_url',
tmp,
)
stream = HDFTradeStream(point=point)
stream.process()
with pandas.HDFStore(os.path.join(tmp, TRADE_FILE_NAME),
'r') as store:
XBTz18 = store['XBTZ18']
assert XBTz18.index[0] == utc_datetime(2018, 12, 5, 0, 0, 25,
948313)
assert XBTz18['side'][0] == SIDE.SELL.value
assert XBTz18['size'][0] == 10000
assert XBTz18['price'][0] == 9.54e-06
assert XBTz18['tickDirection'][
0] == TICK_DIRECTION.ZERO_MINUS_TICK.value
assert XBTz18['grossValue'][0] == 9540000
assert XBTz18['homeNotional'][0] == 10000
assert XBTz18['foreignNotional'][0] == 0.0954
ADAZ18 = store['ADAZ18']
assert ADAZ18.index[0] == utc_datetime(2018, 12, 5, 0, 0, 25,
948313)
assert ADAZ18['side'][0] == SIDE.SELL.value
assert ADAZ18['size'][0] == 166774
assert ADAZ18['price'][0] == 9.54e-06
assert ADAZ18['tickDirection'][
0] == TICK_DIRECTION.MINUS_TICK.value
assert ADAZ18['grossValue'][0] == 159102396
assert ADAZ18['homeNotional'][0] == 166774
assert ADAZ18['foreignNotional'][0] == 1.591024
XBTUSD = store['XBTUSD']
assert XBTUSD.index[0] == utc_datetime(2018, 12, 5, 0, 0, 25,
948313)
assert XBTUSD['side'][0] == SIDE.SELL.value
assert XBTUSD['size'][0] == 11
assert XBTUSD['price'][0] == 9.54e-06
assert XBTUSD['tickDirection'][
0] == TICK_DIRECTION.ZERO_MINUS_TICK.value
assert XBTUSD['grossValue'][0] == 10494
assert XBTUSD['homeNotional'][0] == 11
assert XBTUSD['foreignNotional'][0] == 0.00010494
def test_raw_stream_request() -> None:
with tempfile.TemporaryDirectory() as tmp:
date = utc_datetime(2018, 1, 1)
outcome = os.path.join(tmp, date.strftime("%Y%m%d") + '.csv.gz')
stream = MockRawStreamRequest(point=DatePoint(date, mock_url, tmp),
stream=stream_b)
stream.process()
with open(outcome, 'rb') as f:
content = f.read()
assert content == stream_b
def test_quote_zip_file_stream_exception() -> None:
d = utc_datetime(2018, 1, 1)
with tempfile.TemporaryDirectory() as tmp:
tar_dir = os.path.join(tmp, d.strftime("%Y%m%d"))
stream = MockQuoteZipFileStream(point=DatePoint(date=d,
url=mock_url,
dst_dir=tmp),
stream=stream_quote)
stream._stream_requests = _mock_exception_stream # type:ignore
with pytest.raises(DataDownloadError):
stream.process()
assert not os.path.exists(tar_dir)
def test_trade_hdf_stream_exception() -> None:
d = utc_datetime(2018, 1, 5)
with tempfile.TemporaryDirectory() as tmp:
point = DatePoint(d, 'test_url', tmp)
random_trade_hdf(os.path.join(tmp, TRADE_FILE_NAME), 4)
stream = HDFTradeStream(point=point)
append = random_trade_frame(3, d)
with pytest.raises(DataDownloadError):
with patch("monkq.exchange.bitmex.data.download.read_trade_tar"):
with patch("monkq.exchange.bitmex.data.download.requests"):
with patch(
"monkq.exchange.bitmex.data.download.classify_df"
) as f:
f().items.return_value = [("XBTUSD", append),
("ETHUSD", 's')]
stream.process()
with pandas.HDFStore(os.path.join(tmp, TRADE_FILE_NAME), 'r') as store:
XBT = store['XBTUSD']
assert len(XBT) == 4
def test_bitmex_process_points() -> None:
p = iter(
BitMexProcessPoints(utc_datetime(2018, 1, 1), utc_datetime(2018, 1, 5),
'a', 'b'))
one = next(p)
assert one.value == utc_datetime(2018, 1, 1)
assert one.url == 'a'
assert one.dst_dir == 'b'
two = next(p)
assert two.value == utc_datetime(2018, 1, 2)
assert two.url == 'a'
assert two.dst_dir == 'b'
assert next(p).value == utc_datetime(2018, 1, 3)
assert next(p).value == utc_datetime(2018, 1, 4)
assert next(p).value == utc_datetime(2018, 1, 5)
with pytest.raises(StopIteration):
next(p)
p2 = iter(
BitMexProcessPoints(utc_datetime(2018, 1, 1), utc_datetime(2018, 1, 5),
'a', 'b'))
p_list = list(iter(p2))
assert p_list[0] == DatePoint(utc_datetime(2018, 1, 1), 'a', 'b')
assert p_list[1] == DatePoint(utc_datetime(2018, 1, 2), 'a', 'b')
assert p_list[2] == DatePoint(utc_datetime(2018, 1, 3), 'a', 'b')
assert p_list[3] == DatePoint(utc_datetime(2018, 1, 4), 'a', 'b')
assert p_list[4] == DatePoint(utc_datetime(2018, 1, 5), 'a', 'b')
p3 = iter(
BitMexProcessPoints(utc_datetime(2018, 1, 1), utc_datetime(2018, 1, 1),
'a', 'b'))
assert next(p3).value == utc_datetime(2018, 1, 1)
with pytest.raises(StopIteration):
next(p3)
p4 = iter(
BitMexProcessPoints(utc_datetime(2018, 1, 1), utc_datetime(2018, 1, 1),
'a', 'b'))
p2_list = list(p4)
assert p2_list[0] == DatePoint(utc_datetime(2018, 1, 1), 'a', 'b')
with pytest.raises(IndexError):
p2_list[1]
def test_datepoint() -> None:
d = DatePoint(utc_datetime(2018, 1, 1), 'a', 'b')
assert d.value == utc_datetime(2018, 1, 1)
assert d.url == 'a'
assert d.dst_dir == 'b'