def test_trade_to_1m_kline() -> None: df1 = random_trade_frame(10, utc_datetime(2018, 1, 1, 12, 0, 1)) df2 = random_trade_frame(10, utc_datetime(2018, 1, 2, 12, 0, 1)) df3 = random_trade_frame(10, utc_datetime(2018, 1, 4, 12, 0, 1)) frame = df1.append(df2).append(df3) outcome = trades_to_1m_kline(frame) assert len(outcome) == 4321 assert outcome['high'][0] == max(df1['price']) assert outcome['low'][0] == min(df1['price']) assert outcome['open'][0] == df1['price'][0] assert outcome['close'][0] == df1['price'][-1] assert outcome['volume'][0] == sum(df1['homeNotional']) assert outcome['turnover'][0] == sum(df1['foreignNotional']) assert outcome['high'][20] == max(df1['price']) assert outcome['low'][20] == min(df1['price']) assert outcome['open'][20] == df1['price'][0] assert outcome['close'][20] == df1['price'][-1] assert outcome['volume'][20] == 0 assert outcome['turnover'][20] == 0 assert outcome['high'][-1] == max(df3['price']) assert outcome['low'][-1] == min(df3['price']) assert outcome['open'][-1] == df3['price'][0] assert outcome['close'][-1] == df3['price'][-1] assert outcome['volume'][-1] == sum(df3['homeNotional']) assert outcome['turnover'][-1] == sum(df3['foreignNotional'])
def test_lazy_hdf_table() -> None: store = LazyHDFTableStore(get_resource_path('test_table.hdf')) assert store.cached_table == [] df1 = store.get('XBTZ15') assert df1.index[-1] == utc_datetime(2015, 12, 25, 12) assert df1.iloc[-1]['open'] == 453.11 assert df1.iloc[-1]['close'] == 453.11 assert df1.iloc[-1]['high'] == 453.11 assert df1.iloc[-1]['low'] == 453.11 assert df1.iloc[-1]['volume'] == 650.4802 assert df1.iloc[-1]['turnover'] == 294739.1000 assert df1.index[10] == utc_datetime(2015, 5, 29, 12, 11) assert np.isnan(df1.iloc[10]['open']) assert np.isnan(df1.iloc[10]['high']) assert np.isnan(df1.iloc[10]['close']) assert np.isnan(df1.iloc[10]['low']) assert df1.iloc[10]['volume'] == 0.0 assert df1.iloc[10]['turnover'] == 0.0 assert store.cached_table == ['XBTZ15'] with pytest.raises(DataError): store.get("XBTUSD")
def test_quote_hdf_stream() -> None: with patch("monkq.exchange.bitmex.data.download.requests") as m: resp = m.get() resp.raw = io.BytesIO(stream_quote_gzip) with tempfile.TemporaryDirectory() as tmp: point = DatePoint(utc_datetime(2018, 1, 3), 'test_url', tmp) stream = HDFQuoteStream(point=point) stream.process() with pandas.HDFStore(os.path.join(tmp, QUOTE_FILE_NAME), 'r') as store: XBTz18 = store['XBTZ18'] assert XBTz18.index[0] == utc_datetime(2018, 12, 5, 0, 0, 7, 302458) assert XBTz18['bidSize'][0] == 256083 assert XBTz18['bidPrice'][0] == 9.54e-06 assert XBTz18['askPrice'][0] == 9.55e-06 assert XBTz18['askSize'][0] == 574060 XBTUSD = store['XBTUSD'] assert XBTUSD.index[0] == utc_datetime(2018, 12, 5, 0, 0, 5, 45590) assert XBTUSD['bidSize'][0] == 256081 assert XBTUSD['bidPrice'][0] == 9.54e-06 assert XBTUSD['askPrice'][0] == 9.55e-06 assert XBTUSD['askSize'][0] == 544060 ADAZ18 = store['ADAZ18'] assert ADAZ18.index[0] == utc_datetime(2018, 12, 5, 0, 0, 4, 778522) assert ADAZ18['bidSize'][0] == 256089 assert ADAZ18['bidPrice'][0] == 9.54e-06 assert ADAZ18['askPrice'][0] == 9.55e-06 assert ADAZ18['askSize'][0] == 524060
def test_trade_pickle(future_instrument: FutureInstrument, future_account: FutureAccount) -> None: order_id1 = random_string(6) order = FutureMarketOrder(order_id=order_id1, account=future_account, instrument=future_instrument, quantity=100, submit_datetime=utc_datetime(2018, 1, 1)) trade_id = random_string(6) trade = Trade(order=order, exec_price=13, exec_quantity=50, trade_id=trade_id, trade_datetime=utc_datetime(2018, 1, 1, 0, 1)) p_trade = pickle.dumps(trade) unp_trade = pickle.loads(p_trade) assert unp_trade.order.order_id == order_id1 assert unp_trade.exec_price == 13 assert unp_trade.exec_quantity == 50 assert unp_trade.trade_id == trade_id assert unp_trade.side == SIDE.BUY assert unp_trade.value == 650 assert unp_trade.commission == 1.625 assert unp_trade.trade_datetime == utc_datetime(2018, 1, 1, 0, 1)
def test_runner(tem_data_dir: str) -> None: settings = Setting() custom_settings = { "STRATEGY": TestStrategy, "START_TIME": utc_datetime(2018, 1, 1), "END_TIME": utc_datetime(2018, 2, 1), "RUN_TYPE": RUN_TYPE.BACKTEST, "FREQUENCY": "1m", "DATA_DIR": tem_data_dir, "EXCHANGE": { "bitmex": { 'engine': 'monkq.exchange.bitmex', "IS_TEST": True, "API_KEY": '', "API_SECRET": '', "START_WALLET_BALANCE": 100000 } }, "REPORT_FILE": os.path.join(tem_data_dir, 'result.pkl') } settings.__dict__.update(custom_settings) with over_written_settings(settings, **custom_settings): runner = Runner(settings) runner.run()
def context() -> Generator[MagicMock, None, None]: context = MagicMock() context.exchanges.__getitem__.return_value = MagicMock(BaseSimExchange) context.exchanges.__getitem__( ).all_data.return_value = random_kline_data_with_start_end( utc_datetime(2018, 1, 1), utc_datetime(2018, 2, 1)) context.now = utc_datetime(2018, 1, 10, 0, 1) yield context
def test_analyse_mark_trades(analyse_result: str) -> Figure: analyser = Analyser(analyse_result) fig, axe = analyser.plot_kline('bitmex', '4H', 'XBTZ15', utc_datetime(2015, 5, 15), utc_datetime(2015, 6, 15)) analyser.mark_trades(axe, utc_datetime(2015, 5, 15), utc_datetime(2015, 6, 15)) return fig
def test_kline_indicator() -> None: df_1min = random_kline_data_with_start_end(utc_datetime(2016, 1, 1), utc_datetime(2016, 1, 3), freq='1min') result = kline_indicator(df_1min, 'MA', ['close'], timeperiod=10) assert result[9] == sum(df_1min['close'][:10]) / 10
def process_one_point(self, point: KlinePoint) -> None: # if df is None, it is an end point if point.df is None: if self.cache is not None: kline = trades_to_1m_kline(self.cache) kline.to_hdf(self.output_file, point.key, mode='a', format='table', data_columns=True, index=False, complib=HDF_FILE_COMPRESS_LIB, complevel=HDF_FILE_COMPRESS_LEVEL, append=True) logger.debug("Finished data {}".format(point.key)) # reset everything for another key self.cache = None self.mark_point = START_DATE return end_time = point.df.index[-1] last_date = end_time + relativedelta( hour=0, minute=0, second=0, microsecond=0) logger.debug("Process {} data from {} to {}".format( point.key, self.mark_point, last_date)) if last_date > self.mark_point: process_df = point.df.loc[point.df.index < utc_datetime( last_date.year, last_date.month, last_date.day)] cache_df = point.df.loc[point.df.index >= utc_datetime( last_date.year, last_date.month, last_date.day)] if self.cache is not None: process_df = pandas.concat([self.cache, process_df], copy=False) if len(process_df) != 0: kline = trades_to_1m_kline(process_df) logger.debug( "Write {} data from {} to {} into hdf file".format( point.key, self.mark_point, last_date)) kline.to_hdf(self.output_file, point.key, mode='a', format='table', data_columns=True, index=False, complib=HDF_FILE_COMPRESS_LIB, complevel=HDF_FILE_COMPRESS_LEVEL, append=True) self.cache = cache_df self.mark_point = last_date else: if self.cache is None: self.cache = point.df else: self.cache = pandas.concat([self.cache, point.df], copy=False)
def test_trade_hdf_stream() -> None: with patch("monkq.exchange.bitmex.data.download.requests") as m: resp = m.get() resp.raw = io.BytesIO(stream_trade_gzip) with tempfile.TemporaryDirectory() as tmp: point = DatePoint( utc_datetime(2018, 1, 3), 'test_url', tmp, ) stream = HDFTradeStream(point=point) stream.process() with pandas.HDFStore(os.path.join(tmp, TRADE_FILE_NAME), 'r') as store: XBTz18 = store['XBTZ18'] assert XBTz18.index[0] == utc_datetime(2018, 12, 5, 0, 0, 25, 948313) assert XBTz18['side'][0] == SIDE.SELL.value assert XBTz18['size'][0] == 10000 assert XBTz18['price'][0] == 9.54e-06 assert XBTz18['tickDirection'][ 0] == TICK_DIRECTION.ZERO_MINUS_TICK.value assert XBTz18['grossValue'][0] == 9540000 assert XBTz18['homeNotional'][0] == 10000 assert XBTz18['foreignNotional'][0] == 0.0954 ADAZ18 = store['ADAZ18'] assert ADAZ18.index[0] == utc_datetime(2018, 12, 5, 0, 0, 25, 948313) assert ADAZ18['side'][0] == SIDE.SELL.value assert ADAZ18['size'][0] == 166774 assert ADAZ18['price'][0] == 9.54e-06 assert ADAZ18['tickDirection'][ 0] == TICK_DIRECTION.MINUS_TICK.value assert ADAZ18['grossValue'][0] == 159102396 assert ADAZ18['homeNotional'][0] == 166774 assert ADAZ18['foreignNotional'][0] == 1.591024 XBTUSD = store['XBTUSD'] assert XBTUSD.index[0] == utc_datetime(2018, 12, 5, 0, 0, 25, 948313) assert XBTUSD['side'][0] == SIDE.SELL.value assert XBTUSD['size'][0] == 11 assert XBTUSD['price'][0] == 9.54e-06 assert XBTUSD['tickDirection'][ 0] == TICK_DIRECTION.ZERO_MINUS_TICK.value assert XBTUSD['grossValue'][0] == 10494 assert XBTUSD['homeNotional'][0] == 11 assert XBTUSD['foreignNotional'][0] == 0.00010494
def test_sell_order_trade(instrument: Instrument) -> None: order_id = random_string(6) order = BaseOrder(account=MagicMock(), order_id=order_id, instrument=instrument, quantity=-100, submit_datetime=utc_datetime(2018, 1, 1), text='text2') assert order.order_id == order_id assert order.side == SIDE.SELL assert order.quantity == -100 assert order.order_status == ORDER_STATUS.NOT_TRADED assert order.submit_datetime == utc_datetime(2018, 1, 1) assert order.text == 'text2' trade1 = Trade(order=order, exec_price=10, exec_quantity=-50, trade_id=random_string(6), trade_datetime=utc_datetime(2018, 1, 1, 0, 1)) order.deal(trade1) assert order.traded_quantity == -50 assert order.order_status == ORDER_STATUS.PARTLY_TRADED with pytest.raises(AssertionError): order.deal(trade1) trade2 = Trade(order=order, exec_price=11, exec_quantity=-50, trade_id=random_string(6), trade_datetime=utc_datetime(2018, 1, 1, 0, 2)) order.deal(trade2) assert order.traded_quantity == -100 assert order.order_status == ORDER_STATUS.FULL_TRADED assert trade1 in order.trades assert trade2 in order.trades assert trade1.order_id == order_id assert trade1.side == SIDE.SELL assert trade1.exec_price == 10 assert trade1.value == -500 assert trade1.commission == 1.25 assert trade2.order_id == order_id assert trade2.side == SIDE.SELL assert trade2.exec_price == 11 assert trade2.value == -550 assert trade2.commission == 1.375
def test_initer(context: MagicMock) -> None: initer = Initer(context) instrument = MagicMock() initer.init_kline_freq('30min', instrument) initer.init_indicator('30min', instrument, 'MA', "MA_store", ['close']) initer.init_indicator('1min', instrument, "MA", "ma1m", ['close']) history_kline_30min = initer.history_kline('30min', instrument, 100) assert len(history_kline_30min) == 100 assert history_kline_30min.index[-1] == utc_datetime(2018, 1, 10) history_ma_30min = initer.history_indicator("MA_store", 100) assert len(history_ma_30min) == 100 assert history_ma_30min.index[-1] == utc_datetime(2018, 1, 10)
def test_quote_zip_file_stream() -> None: d = utc_datetime(2018, 1, 1) with tempfile.TemporaryDirectory() as tmp: tar_dir = os.path.join(tmp, d.strftime("%Y%m%d")) stream = MockQuoteZipFileStream(point=DatePoint(date=d, url=mock_url, dst_dir=tmp), stream=stream_quote) stream.process() with gzip.open(os.path.join(tar_dir, "ADAZ18.csv.gz")) as f: content = f.read() assert content == b"""timestamp,symbol,bidSize,bidPrice,askPrice,askSize 2018-12-05D00:00:04.778522000,ADAZ18,256089,9.54e-06,9.55e-06,524060 """ with gzip.open(os.path.join(tar_dir, "XBTUSD.csv.gz")) as f: content = f.read() assert content == b"""timestamp,symbol,bidSize,bidPrice,askPrice,askSize 2018-12-05D00:00:05.045590000,XBTUSD,256081,9.54e-06,9.55e-06,544060 """ with gzip.open(os.path.join(tar_dir, "XBTZ18.csv.gz")) as f: content = f.read() assert content == b"""timestamp,symbol,bidSize,bidPrice,askPrice,askSize
def test_trade_zip_file_stream() -> None: d = utc_datetime(2018, 1, 1) with tempfile.TemporaryDirectory() as tmp: tar_dir = os.path.join(tmp, d.strftime("%Y%m%d")) stream = MockTradeZipFileStream(point=DatePoint(date=d, url=mock_url, dst_dir=tmp), stream=stream_trade) stream.process() with gzip.open(os.path.join(tar_dir, "ADAZ18.csv.gz")) as f: content = f.read() assert content == b"""timestamp,symbol,side,size,price,tickDirection,trdMatchID,grossValue,homeNotional,foreignNotional 2018-12-05D00:00:25.948313000,ADAZ18,Sell,166774,9.54e-06,MinusTick,498350a3-8c03-affd-cf87-e6d8a90b70e5,159102396,166774,1.591024 """ with gzip.open(os.path.join(tar_dir, "XBTUSD.csv.gz")) as f: content = f.read() assert content == b"""timestamp,symbol,side,size,price,tickDirection,trdMatchID,grossValue,homeNotional,foreignNotional 2018-12-05D00:00:25.948313000,XBTUSD,Sell,11,9.54e-06,ZeroMinusTick,1d004a95-44fa-c75c-47cd-d651ecd41c25,10494,11,0.00010494 """ with gzip.open(os.path.join(tar_dir, "XBTZ18.csv.gz")) as f: content = f.read() assert content == b"""timestamp,symbol,side,size,price,tickDirection,trdMatchID,grossValue,homeNotional,foreignNotional
def test_trader_counter() -> None: stat = MagicMock() exchange = MagicMock() account = MagicMock() account.exchange = exchange trade_counter = TradeCounter(stat) exchange.last_price.return_value = 20. order1 = LimitOrder(account=account, order_id=gen_unique_id(), instrument=MagicMock(), quantity=100, price=10) order2 = LimitOrder(account=account, order_id=gen_unique_id(), instrument=MagicMock(), quantity=200, price=20) order3 = MarketOrder(account=account, order_id=gen_unique_id(), instrument=MagicMock(), quantity=100) trade_counter.submit_order(order1) trade_counter.submit_order(order2) trade_counter.submit_order(order3) assert len(trade_counter.open_orders()) == 3 trade_counter.cancel_order(order1.order_id) with pytest.raises(NotImplementedError): trade_counter.amend_order(order2.order_id, 100, 20) assert len(trade_counter.open_orders()) == 2 trade_counter.match(utc_datetime(2018, 1, 1)) assert len(order2.trades) == 1 assert len(order3.trades) == 1 assert len(trade_counter.open_orders()) == 0 trades = [] trades.extend(order2.trades) trades.extend(order3.trades) trade_calls = [call(t) for t in trades] account.deal.assert_has_calls(trade_calls)
def test_fullfill_kline_func() -> None: k_df1 = random_kline_data(1000, utc_datetime(2018, 1, 2, 12, 30)) k_df2 = random_kline_data(1000, utc_datetime(2018, 1, 6, 12, 30)) kline_df = k_df1.append(k_df2) full_kline = fullfill_1m_kline_with_start_end(kline_df, utc_datetime(2018, 1, 1, 12, 1), utc_datetime(2018, 1, 20, 12)) assert len(full_kline) == 27360 compare_dataframe_time(full_kline, k_df1, utc_datetime(2018, 1, 2, 12, 29)) compare_dataframe_filled(full_kline, utc_datetime(2018, 1, 2, 12, 35), k_df1, utc_datetime(2018, 1, 2, 12, 30)) compare_dataframe_time(full_kline, k_df2, utc_datetime(2018, 1, 6, 12, 25)) compare_dataframe_filled(full_kline, utc_datetime(2018, 1, 6, 12, 35), k_df2, utc_datetime(2018, 1, 6, 12, 30))
def test_check_1m_data_integrity() -> None: df1 = random_kline_data(10, utc_datetime(2018, 1, 1, 12, 10)) assert check_1m_data_integrity(df1, utc_datetime(2018, 1, 1, 12), utc_datetime(2018, 1, 1, 12, 10)) assert not check_1m_data_integrity(df1, utc_datetime(2018, 1, 1, 12, 1), utc_datetime(2018, 1, 1, 12, 10)) assert not check_1m_data_integrity(df1, utc_datetime(2018, 1, 1, 12), utc_datetime(2018, 1, 1, 12, 12))
def __init__(self, context: "Context"): self.context = context self.report_file: str = getattr(self.context.settings, 'REPORT_FILE', 'result.pkl') self.collect_freq = getattr(self.context.settings, 'COLLECT_FREQ', '4H') self.daily_capital: List[DAILY_STAT_TYPE] = [] self.order_collections: List[BaseOrder] = [] self.trade_collections: List[Trade] = [] self.collect_offset: DateOffset = to_offset(self.collect_freq) self.last_collect_time: datetime.datetime = utc_datetime(1970, 1, 1)
def test_bitmex_dataloader_kline_data(exchange: MagicMock, tem_data_dir: str) -> None: context = MagicMock() dataloader = BitmexDataloader(tem_data_dir) instrument = MagicMock() instrument.symbol = "XBTZ15" context.now = utc_datetime(2015, 12, 25, 11, 49) assert dataloader.get_last_price(instrument.symbol, context.now) == 453.5 kline_df = dataloader.get_kline(instrument.symbol, context.now, 50) assert len(kline_df) == 50 assert kline_df.index[-1] == utc_datetime(2015, 12, 25, 11, 49) context.now = utc_datetime(2016, 1, 1, 11, 12) assert dataloader.get_last_price(instrument.symbol, context.now) == 0 dataloader.all_data(instrument.symbol)
def test_raw_stream_request_exception() -> None: with tempfile.TemporaryDirectory() as tmp: date = utc_datetime(2018, 1, 1) outcome = os.path.join(tmp, date.strftime("%Y%m%d") + '.csv.gz') stream = MockRawStreamRequest(point=DatePoint(date, mock_url, tmp), stream=stream_b) stream._stream_requests = _mock_exception_stream # type:ignore with pytest.raises(DataDownloadError): stream.process() assert not os.path.exists(outcome)
def test_symbols_stream_request() -> None: with tempfile.TemporaryDirectory() as tmp: stream = MockSymbolsStream(point=DatePoint(utc_datetime(2018, 1, 1), mock_url, tmp), stream=stream_symbols) stream.process() with open(os.path.join(tmp, INSTRUMENT_FILENAME), 'rb') as f: content = f.read() assert content == stream_symbols
def test_raw_stream_request() -> None: with tempfile.TemporaryDirectory() as tmp: date = utc_datetime(2018, 1, 1) outcome = os.path.join(tmp, date.strftime("%Y%m%d") + '.csv.gz') stream = MockRawStreamRequest(point=DatePoint(date, mock_url, tmp), stream=stream_b) stream.process() with open(outcome, 'rb') as f: content = f.read() assert content == stream_b
async def test_bitmex_exchange_simulate(tem_data_dir: str, instrument: FutureInstrument) -> None: context = MagicMock() account = MagicMock() trade_counter = TradeCounter(MagicMock()) context.trade_counter = trade_counter context.settings.DATA_DIR = tem_data_dir sim_exchange = BitmexSimulateExchange(context, 'bitmex', {"START_WALLET_BALANCE": 1000000}) await sim_exchange.setup() instrument = instrument context.now = utc_datetime(2016, 10, 3, 12, 30) assert await sim_exchange.get_last_price(instrument) == 63744.0 assert sim_exchange.exchange_info() == bitmex_info order_id = await sim_exchange.place_limit_order(account, instrument, 10, 100, 'order_text') market_order_id = await sim_exchange.place_market_order(account, instrument, 100, 'order_text2') await sim_exchange.cancel_order(account, order_id) open_orders = await sim_exchange.open_orders(account) assert len(open_orders) == 1 order = open_orders[0] assert order['order_id'] == market_order_id assert await sim_exchange.available_instruments() ins = await sim_exchange.get_instrument('XBUZ15') assert ins.symbol == 'XBUZ15' kline = await sim_exchange.get_kline(instrument) assert kline.index[-1] == utc_datetime(2016, 10, 3, 12, 30) sim_exchange.match_open_orders() sim_exchange.all_data(instrument)
def test_run_1m_backtest(start_strategy_condition: str) -> None: from manage import cmd_main as strategy_cmd strategy_cmd.main(['runstrategy'], standalone_mode=False) with open(os.path.join(start_strategy_condition, 'result.pkl'), 'rb') as f: obj = pickle.load(f) daily_capital = obj['daily_capital'] assert pytest.approx(daily_capital[-1]['bitmex_account'], 113324.03) assert daily_capital[-1]['timestamp'] == utc_datetime(2015, 12, 1) assert pytest.approx(daily_capital[1]['bitmex_account'], 98721.99024999999) assert daily_capital[1]['timestamp'] == utc_datetime(2015, 6, 2) assert len(obj['orders']) == 1 order = obj['orders'][0] assert order.instrument.symbol == "XBTZ15" assert len(obj['trades']) == 1 trade = obj['trades'][0] assert trade.order.instrument.symbol == "XBTZ15"
def test_statistic(statistic_context: MagicMock) -> None: stat = Statistic(statistic_context) stat.freq_collect_account() assert stat.daily_capital == [{"timestamp": utc_datetime(2018, 1, 1), "account1": 2000, "account2": 5000}] order = PickleMock() stat.collect_order(order) # type:ignore assert order in stat.order_collections trade = PickleMock() stat.collect_trade(trade) # type:ignore assert trade in stat.trade_collections stat.report() with open(stat.report_file, 'rb') as f: obj = pickle.load(f) assert obj['daily_capital'] == [{"timestamp": utc_datetime(2018, 1, 1), "account1": 2000, "account2": 5000}]
def test_kline_time_window() -> None: df_1min = random_kline_data_with_start_end(utc_datetime(2016, 1, 1), utc_datetime(2016, 1, 10), freq='1min') result = kline_time_window(df_1min, utc_datetime(2016, 1, 2), utc_datetime(2016, 1, 9)) assert result.index[0] == utc_datetime(2016, 1, 2) assert result.index[-1] == utc_datetime(2016, 1, 9) day = random.randint(2, 8) hour = random.randint(0, 23) minute = random.randint(0, 59) assert result.loc[utc_datetime(2016, 1, day, hour, minute)]['close'] == \ df_1min.loc[utc_datetime(2016, 1, day, hour, minute)]['close'] # noqa:E128
def statistic_context(settings: Setting) -> Generator[PickleMock, None, None]: account1 = PickleMock() account2 = PickleMock() account1.total_capital = 2000 # type:ignore account2.total_capital = 5000 # type:ignore accounts = {"account1": account1, "account2": account2} context = MagicMock() context.now = utc_datetime(2018, 1, 1) context.accounts = accounts context.settings = settings with tempfile.TemporaryDirectory() as tmp: setattr(context.settings, 'REPORT_FILE', os.path.join(tmp, 'result.pkl')) yield context
def test_quote_zip_file_stream_exception() -> None: d = utc_datetime(2018, 1, 1) with tempfile.TemporaryDirectory() as tmp: tar_dir = os.path.join(tmp, d.strftime("%Y%m%d")) stream = MockQuoteZipFileStream(point=DatePoint(date=d, url=mock_url, dst_dir=tmp), stream=stream_quote) stream._stream_requests = _mock_exception_stream # type:ignore with pytest.raises(DataDownloadError): stream.process() assert not os.path.exists(tar_dir)
def test_trade_hdf_stream_exception() -> None: d = utc_datetime(2018, 1, 5) with tempfile.TemporaryDirectory() as tmp: point = DatePoint(d, 'test_url', tmp) random_trade_hdf(os.path.join(tmp, TRADE_FILE_NAME), 4) stream = HDFTradeStream(point=point) append = random_trade_frame(3, d) with pytest.raises(DataDownloadError): with patch("monkq.exchange.bitmex.data.download.read_trade_tar"): with patch("monkq.exchange.bitmex.data.download.requests"): with patch( "monkq.exchange.bitmex.data.download.classify_df" ) as f: f().items.return_value = [("XBTUSD", append), ("ETHUSD", 's')] stream.process() with pandas.HDFStore(os.path.join(tmp, TRADE_FILE_NAME), 'r') as store: XBT = store['XBTUSD'] assert len(XBT) == 4
def get_start(cls, dst_dir: str) -> datetime.datetime: if cls.kind == 'quote': filename = QUOTE_FILE_NAME elif cls.kind == 'trade': filename = TRADE_FILE_NAME else: raise Exception("kind should be quote or trade") try: with pandas.HDFStore(os.path.join(dst_dir, filename), 'r') as store: keys = store.keys() max_date = START_DATE for key in keys: index = store.select_column(key, 'index') last = max(index) max_date = max(max_date, last) last_date = utc_datetime(max_date.year, max_date.month, max_date.day) return last_date + relativedelta(days=+1) except (KeyError, OSError): return START_DATE