def _get_test_case_set_up():
timer = SettableTimer()
tickers = [QuandlTicker("MSFT", "WIKI"), QuandlTicker("AAPL", "WIKI")]
price_data_provider_mock = _create_price_provider_mock(tickers)
data_handler = DailyDataHandler(price_data_provider_mock, timer)
return timer, tickers, data_handler
def setUp(self):
try:
self.data_provider = get_data_provider()
except Exception as e:
raise self.skipTest(e)
self.ticker_1.initialize_data_provider(self.timer, self.data_provider)
self.ticker_2.initialize_data_provider(self.timer, self.data_provider)
self.data_handler = DailyDataHandler(self.data_provider, self.timer)
self.timer.set_current_time(str_to_date("2017-12-20 00:00:00.000000", DateFormat.FULL_ISO))
def setUp(self):
try:
self.price_data_provider = get_data_provider()
except Exception as e:
raise self.skipTest(e)
self.timer = SettableTimer()
self.data_handler = DailyDataHandler(self.price_data_provider,
self.timer)
MarketOpenEvent.set_trigger_time({
"hour": 13,
"minute": 30,
"second": 0,
"microsecond": 0
})
MarketCloseEvent.set_trigger_time({
"hour": 20,
"minute": 0,
"second": 0,
"microsecond": 0
})
def _setup_data_handler(self, volume_value: Optional[float]) -> DataHandler:
dates = pd.date_range(str_to_date("2019-12-01"), str_to_date("2020-01-30"), freq='D')
prices_data_frame = QFDataFrame(data={PriceField.Volume: [volume_value] * len(dates)},
index=dates)
prices_data_array = tickers_dict_to_data_array({
self.ticker: prices_data_frame,
}, [self.ticker], [PriceField.Volume])
data_provider = PresetDataProvider(prices_data_array, dates[0], dates[-1], Frequency.DAILY)
timer = SettableTimer(dates[-1])
return DailyDataHandler(data_provider, timer)
def _create_data_handler(self, data_provider, timer):
if self._frequency == Frequency.MIN_1:
data_handler = IntradayDataHandler(data_provider, timer)
elif self._frequency == Frequency.DAILY:
data_handler = DailyDataHandler(data_provider, timer)
else:
raise ValueError(
"Invalid frequency parameter. The only frequencies supported by the DataHandler are "
"Frequency.DAILY and Frequency.MIN_1. "
"\nMake sure you set the frequency in the session builder for example: "
"\n\t-> 'session_builder.set_frequency(Frequency.DAILY)'")
return data_handler
def setUp(self):
self.timer = SettableTimer()
self.tickers = [
QuandlTicker("MSFT", "WIKI"),
QuandlTicker("AAPL", "WIKI")
]
price_data_provider_mock = self._create_price_provider_mock(
self.tickers)
self.data_handler = DailyDataHandler(price_data_provider_mock,
self.timer)
MarketOpenEvent.set_trigger_time({
"hour": 13,
"minute": 30,
"second": 0,
"microsecond": 0
})
MarketCloseEvent.set_trigger_time({
"hour": 20,
"minute": 0,
"second": 0,
"microsecond": 0
})
def setUp(self):
try:
self.data_provider = get_data_provider()
except Exception as e:
raise self.skipTest(e)
self.TICKER_1.initialize_data_provider(self.timer, self.data_provider)
self.TICKER_2.initialize_data_provider(self.timer, self.data_provider)
data_provider = PrefetchingDataProvider(self.data_provider,
self.TICKER_2,
PriceField.ohlcv(),
self.start_date, self.end_date,
self.frequency)
self.timer.set_current_time(self.end_date)
self.data_handler = DailyDataHandler(data_provider, self.timer)
class TestDailyDataHandler(TestCase):
def setUp(self):
self.timer = SettableTimer()
self.tickers = [
QuandlTicker("MSFT", "WIKI"),
QuandlTicker("AAPL", "WIKI")
]
price_data_provider_mock = self._create_price_provider_mock(
self.tickers)
self.data_handler = DailyDataHandler(price_data_provider_mock,
self.timer)
MarketOpenEvent.set_trigger_time({
"hour": 13,
"minute": 30,
"second": 0,
"microsecond": 0
})
MarketCloseEvent.set_trigger_time({
"hour": 20,
"minute": 0,
"second": 0,
"microsecond": 0
})
def test_before_data_starts(self):
self._assert_last_prices_are_correct("2009-12-28 06:00:00.000000",
[nan, nan])
def test_at_market_open_when_data_available_for_every_ticker(self):
self._assert_last_prices_are_correct("2009-12-28 13:30:00.000000",
[25.0, 27.0])
def test_at_market_open_when_data_available_for_the_first_ticker(self):
self._assert_last_prices_are_correct("2009-12-30 13:30:00.000000",
[31.0, 30.0])
def test_during_trading_hours_when_data_available_for_every_ticker(self):
self._assert_last_prices_are_correct("2009-12-28 14:30:00.000000",
[25.0, 27.0])
def test_during_trading_hours_when_data_available_for_the_first_ticker(
self):
self._assert_last_prices_are_correct("2009-12-30 14:30:00.000000",
[31.0, 30.0])
def test_at_market_close_when_data_available_for_the_first_ticker(self):
self._assert_last_prices_are_correct("2009-12-30 20:00:00.000000",
[32.0, 30.0])
def test_after_market_close_when_data_is_not_available_for_anything_anymore(
self):
self._assert_last_prices_are_correct("2009-12-30 20:00:00.000000",
[32.0, 30.0])
def test_before_market_close_when_old_data_is_available(self):
self._assert_last_prices_are_correct("2009-12-31 06:00:00.000000",
[32.0, 30.0])
def _assert_last_prices_are_correct(self, curr_time_str, expected_values):
current_time = str_to_date(curr_time_str, DateFormat.FULL_ISO)
self.timer.set_current_time(current_time)
expected_series = QFSeries(data=expected_values, index=self.tickers)
actual_series = self.data_handler.get_last_available_price(
self.tickers)
assert_series_equal(expected_series, actual_series, check_names=False)
def _create_price_provider_mock(self, tickers):
mock_data_array = QFDataArray.create(
dates=date_range(start='2009-12-28', end='2009-12-30', freq='D'),
tickers=tickers,
fields=PriceField.ohlcv(),
data=[
# 2009-12-28
[
# Open High Low Close Volume
[25.0, 25.1, 25.2, 26.0, 25.3], # MSFT
[27.0, 27.1, 27.2, 28.0, 27.3] # AAPL
],
# 2009-12-29
[
# Open High Low Close Volume
[None, None, None, None, None], # MSFT
[29.0, 29.1, 29.2, 30.0, 29.3] # AAPL
],
# 2009-12-30
[
# Open High Low Close Volume
[31.0, 31.1, 31.2, 32.0, 31.3], # MSFT
[None, None, None, None, None] # AAPL
]
])
price_data_provider_mock = Mock(spec=DataProvider,
frequency=Frequency.DAILY)
price_data_provider_mock.get_price.side_effect = lambda t, fields, start_time, end_time, frequency: \
mock_data_array.loc[start_time:end_time, t, fields].to_pandas()
return price_data_provider_mock
class TestDataHandler(TestCase):
@classmethod
def setUpClass(cls):
cls.spx_index_ticker = BloombergTicker("SPX Index")
cls.google_ticker = BloombergTicker("GOOGL US Equity")
cls.microsoft_ticker = BloombergTicker("MSFT US Equity")
cls.start_date = str_to_date("2018-01-02")
cls.end_date = str_to_date("2018-01-31")
cls.end_date_trimmed = str_to_date("2018-01-30")
cls.get_history_field = "PX_TO_BOOK_RATIO"
def setUp(self):
try:
self.price_data_provider = get_data_provider()
except Exception as e:
raise self.skipTest(e)
self.timer = SettableTimer()
self.data_handler = DailyDataHandler(self.price_data_provider,
self.timer)
MarketOpenEvent.set_trigger_time({
"hour": 13,
"minute": 30,
"second": 0,
"microsecond": 0
})
MarketCloseEvent.set_trigger_time({
"hour": 20,
"minute": 0,
"second": 0,
"microsecond": 0
})
def test_get_price_when_end_date_is_in_the_past(self):
self.timer.set_current_time(
str_to_date("2018-02-12 00:00:00.000000", DateFormat.FULL_ISO))
prices_tms = self.data_handler.get_price(self.spx_index_ticker,
PriceField.Close,
self.start_date,
self.end_date)
self.assertEqual(self.start_date, prices_tms.index[0].to_pydatetime())
self.assertEqual(self.end_date, prices_tms.index[-1].to_pydatetime())
def test_get_price_when_end_date_is_today_after_market_close(self):
self.timer.set_current_time(
str_to_date("2018-01-31") + MarketCloseEvent.trigger_time() +
RelativeDelta(hours=1))
prices_tms = self.data_handler.get_price(self.spx_index_ticker,
PriceField.Close,
self.start_date,
self.end_date)
self.assertEqual(self.start_date, prices_tms.index[0].to_pydatetime())
self.assertEqual(self.end_date, prices_tms.index[-1].to_pydatetime())
def test_get_price_when_end_date_is_today_before_market_close(self):
self.timer.set_current_time(
str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() +
RelativeDelta(hours=1))
close_prices_tms = self.data_handler.get_price(self.spx_index_ticker,
PriceField.Close,
self.start_date,
self.end_date)
self.assertEqual(self.start_date,
close_prices_tms.index[0].to_pydatetime())
self.assertEqual(self.end_date_trimmed,
close_prices_tms.index[-1].to_pydatetime())
def test_get_open_price_when_end_date_is_today_before_market_close__single_ticker(
self):
self.timer.set_current_time(
str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() +
RelativeDelta(hours=1))
current_date = datetime(self.timer.now().year,
self.timer.now().month,
self.timer.now().day)
open_prices_tms = self.data_handler.get_price(self.spx_index_ticker,
PriceField.Open,
self.start_date,
self.timer.now())
self.assertEqual(self.start_date,
open_prices_tms.index[0].to_pydatetime())
self.assertEqual(current_date,
open_prices_tms.index[-1].to_pydatetime())
prices_df = self.data_handler.get_price(
self.spx_index_ticker, [PriceField.Open, PriceField.Close],
self.start_date, self.timer.now())
self.assertEqual(self.start_date, prices_df.index[0].to_pydatetime())
self.assertEqual(current_date, prices_df.index[-1].to_pydatetime())
last_bar = prices_df.iloc[-1]
self.assertTrue(isnan(last_bar.loc[PriceField.Close]))
self.assertIsNotNone(last_bar.loc[PriceField.Open])
def test_get_open_price_when_end_date_is_today_before_market_close__multiple_tickers(
self):
self.timer.set_current_time(
str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() +
RelativeDelta(hours=1))
current_date = datetime(self.timer.now().year,
self.timer.now().month,
self.timer.now().day)
tickers = [self.spx_index_ticker, self.microsoft_ticker]
open_prices_tms = self.data_handler.get_price(tickers, PriceField.Open,
self.start_date,
self.timer.now())
self.assertEqual(self.start_date,
open_prices_tms.index[0].to_pydatetime())
self.assertEqual(current_date,
open_prices_tms.index[-1].to_pydatetime())
prices_data_array = self.data_handler.get_price(
tickers, [PriceField.Open, PriceField.Close], self.start_date,
self.timer.now())
last_bar_df = prices_data_array.loc[current_date, :, :].to_pandas()
self.assertTrue(last_bar_df[PriceField.Close].isna().all())
self.assertTrue(last_bar_df[PriceField.Open].notna().all())
def test_get_price_when_end_date_is_tomorrow(self):
self.timer.set_current_time(
str_to_date("2018-01-30") + MarketCloseEvent.trigger_time() +
RelativeDelta(hours=1))
prices_tms = self.data_handler.get_price(self.spx_index_ticker,
PriceField.Close,
self.start_date,
self.end_date_trimmed)
self.assertEqual(self.start_date, prices_tms.index[0].to_pydatetime())
self.assertEqual(self.end_date_trimmed,
prices_tms.index[-1].to_pydatetime())
def test_get_last_price_single_ticker(self):
# just test if getting single ticker value works, when a single ticker (not wrapped in a list) is passed
self.timer.set_current_time(
str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() +
RelativeDelta(hours=1))
single_price = self.data_handler.get_last_available_price(
self.spx_index_ticker)
self.assertTrue(isinstance(single_price, float))
# at market open
self.timer.set_current_time(
str_to_date("2018-01-31") + MarketOpenEvent.trigger_time())
at_market_open = self.data_handler.get_last_available_price(
[self.spx_index_ticker])
self.assertEqual(self.spx_index_ticker, at_market_open.index[0])
self.assertEqual(single_price, at_market_open[0])
# during the trading session
self.timer.set_current_time(
str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() +
RelativeDelta(hours=1))
during_the_day_last_prices = self.data_handler.get_last_available_price(
[self.spx_index_ticker])
self.assertEqual(self.spx_index_ticker,
during_the_day_last_prices.index[0])
self.assertEqual(single_price, during_the_day_last_prices[0])
# after the trading session
self.timer.set_current_time(
str_to_date("2018-01-31") + MarketCloseEvent.trigger_time() +
RelativeDelta(hours=1))
after_close_last_prices = self.data_handler.get_last_available_price(
[self.spx_index_ticker])
self.assertEqual(self.spx_index_ticker,
after_close_last_prices.index[0])
self.assertNotEqual(during_the_day_last_prices[0],
after_close_last_prices[0])
# before the trading session
self.timer.set_current_time(
str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() -
RelativeDelta(hours=1))
before_trading_session_prices = self.data_handler.get_last_available_price(
[self.spx_index_ticker])
self.assertEqual(self.spx_index_ticker,
before_trading_session_prices.index[0])
self.assertNotEqual(during_the_day_last_prices[0],
before_trading_session_prices[0])
self.assertNotEqual(after_close_last_prices[0],
before_trading_session_prices[0])
def test_get_last_price_with_multiple_tickers_when_current_data_is_unavailable(
self):
self.timer.set_current_time(
str_to_date("2018-01-01") + MarketOpenEvent.trigger_time() +
RelativeDelta(hours=1))
last_prices = self.data_handler.get_last_available_price(
[self.spx_index_ticker, self.google_ticker])
self.assertEqual(self.spx_index_ticker, last_prices.index[0])
self.assertEqual(self.google_ticker, last_prices.index[1])
def test_get_last_price_with_empty_tickers_list(self):
self.timer.set_current_time(
str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() +
RelativeDelta(hours=1))
last_prices = self.data_handler.get_last_available_price([])
assert_series_equal(QFSeries(), last_prices)
def test_get_history_when_end_date_is_in_the_past(self):
self.timer.set_current_time(
str_to_date("2018-02-12 00:00:00.000000", DateFormat.FULL_ISO))
prices_tms = self.data_handler.get_history(self.spx_index_ticker,
self.get_history_field,
self.start_date,
self.end_date)
self.assertEqual(self.start_date, prices_tms.index[0].to_pydatetime())
self.assertEqual(self.end_date, prices_tms.index[-1].to_pydatetime())
def test_get_history_when_end_date_is_today_after_market_close(self):
self.timer.set_current_time(
str_to_date("2018-01-31") + MarketCloseEvent.trigger_time() +
RelativeDelta(hours=1))
prices_tms = self.data_handler.get_history(self.spx_index_ticker,
self.get_history_field,
self.start_date,
self.end_date)
self.assertEqual(self.start_date, prices_tms.index[0].to_pydatetime())
self.assertEqual(self.end_date, prices_tms.index[-1].to_pydatetime())
def test_get_history_when_end_date_is_today_before_market_close(self):
self.timer.set_current_time(
str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() +
RelativeDelta(hours=1))
prices_tms = self.data_handler.get_history(self.spx_index_ticker,
self.get_history_field,
self.start_date,
self.end_date)
self.assertEqual(self.start_date, prices_tms.index[0].to_pydatetime())
self.assertEqual(self.end_date_trimmed,
prices_tms.index[-1].to_pydatetime())
def test_get_history_when_end_date_is_tomorrow(self):
self.timer.set_current_time(
str_to_date("2018-01-30") + MarketCloseEvent.trigger_time() +
RelativeDelta(hours=1))
prices_tms = self.data_handler.get_history(self.spx_index_ticker,
self.get_history_field,
self.start_date,
self.end_date_trimmed)
self.assertEqual(self.start_date, prices_tms.index[0].to_pydatetime())
self.assertEqual(self.end_date_trimmed,
prices_tms.index[-1].to_pydatetime())
def test_get_history_with_multiple_tickers(self):
self.timer.set_current_time(
str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() +
RelativeDelta(hours=1))
resilt_df = self.data_handler.get_history(
[self.microsoft_ticker, self.google_ticker],
self.get_history_field, self.start_date, self.end_date_trimmed)
self.assertEqual(self.microsoft_ticker, resilt_df.columns[0])
self.assertEqual(self.google_ticker, resilt_df.columns[1])
self.assertEqual(self.start_date, resilt_df.index[0].to_pydatetime())
self.assertEqual(self.end_date_trimmed,
resilt_df.index[-1].to_pydatetime())
self.assertEqual(resilt_df.shape, (20, 2))
def test_historical_price_many_tickers_many_fields(self):
self.timer.set_current_time(
str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() +
RelativeDelta(hours=1))
result_array = self.data_handler.historical_price(
[self.microsoft_ticker], [PriceField.Open, PriceField.Close],
nr_of_bars=5)
self.assertEqual(QFDataArray, type(result_array))
self.assertEqual((5, 1, 2), result_array.shape)
expected_dates_str = [
"2018-01-24", "2018-01-25", "2018-01-26", "2018-01-29",
"2018-01-30"
]
expected_dates = [
str_to_date(date_str) for date_str in expected_dates_str
]
assert_same_index(pd.DatetimeIndex(expected_dates, name=DATES),
result_array.dates.to_index(),
check_index_type=True,
check_names=True)
def test_historical_price_many_tickers_one_field(self):
self.timer.set_current_time(
str_to_date("2018-01-04") + MarketOpenEvent.trigger_time() +
RelativeDelta(hours=1))
result_df = self.data_handler.historical_price([self.microsoft_ticker],
PriceField.Open,
nr_of_bars=5)
self.assertEqual(PricesDataFrame, type(result_df))
expected_dates_idx = pd.DatetimeIndex([
'2017-12-27', '2017-12-28', '2017-12-29', '2018-01-02',
'2018-01-03'
],
name=DATES)
assert_same_index(expected_dates_idx,
result_df.index,
check_index_type=True,
check_names=True)
expected_tickers_idx = pd.Index([self.microsoft_ticker], name=TICKERS)
assert_same_index(expected_tickers_idx,
result_df.columns,
check_index_type=True,
check_names=True)
def __init__(self, data_provider: DataProvider, start_date, end_date, initial_cash,
frequency: Frequency = Frequency.MIN_1):
"""
Set up the backtest variables according to what has been passed in.
"""
super().__init__()
self.logger = qf_logger.getChild(self.__class__.__name__)
self.logger.info(
"\n".join([
"Testing the Backtester:",
"Start date: {:s}".format(date_to_str(start_date)),
"End date: {:s}".format(date_to_str(end_date)),
"Initial cash: {:.2f}".format(initial_cash),
"Frequency of the simulated execution handler: {}".format(frequency)
])
)
timer = SettableTimer(start_date)
notifiers = Notifiers(timer)
if frequency <= Frequency.DAILY:
data_handler = DailyDataHandler(data_provider, timer)
else:
data_handler = IntradayDataHandler(data_provider, timer)
contract_ticker_mapper = SimulatedBloombergContractTickerMapper()
portfolio = Portfolio(data_handler, initial_cash, timer, contract_ticker_mapper)
signals_register = BacktestSignalsRegister()
backtest_result = BacktestResult(portfolio=portfolio, backtest_name="Testing the Backtester",
start_date=start_date, end_date=end_date, signals_register=signals_register)
monitor = Mock(spec=BacktestMonitor)
commission_model = FixedCommissionModel(0.0)
slippage_model = PriceBasedSlippage(0.0, data_provider, contract_ticker_mapper)
execution_handler = SimulatedExecutionHandler(
data_handler, timer, notifiers.scheduler, monitor, commission_model,
contract_ticker_mapper, portfolio, slippage_model)
broker = BacktestBroker(portfolio, execution_handler)
order_factory = OrderFactory(broker, data_handler, contract_ticker_mapper)
event_manager = self._create_event_manager(timer, notifiers)
time_flow_controller = BacktestTimeFlowController(
notifiers.scheduler, event_manager, timer, notifiers.empty_queue_event_notifier, end_date
)
position_sizer = SimplePositionSizer(broker, data_handler, order_factory, contract_ticker_mapper,
signals_register)
self.logger.info(
"\n".join([
"Configuration of components:",
"Position sizer: {:s}".format(position_sizer.__class__.__name__),
"Timer: {:s}".format(timer.__class__.__name__),
"Data Provider: {:s}".format(data_provider.__class__.__name__),
"Backtest Result: {:s}".format(backtest_result.__class__.__name__),
"Monitor: {:s}".format(monitor.__class__.__name__),
"Execution Handler: {:s}".format(execution_handler.__class__.__name__),
"Commission Model: {:s}".format(commission_model.__class__.__name__),
"Broker: {:s}".format(broker.__class__.__name__),
"Contract-Ticker Mapper: {:s}".format(contract_ticker_mapper.__class__.__name__)
])
)
self.broker = broker
self.notifiers = notifiers
self.initial_cash = initial_cash
self.start_date = start_date
self.end_date = end_date
self.event_manager = event_manager
self.contract_ticker_mapper = contract_ticker_mapper
self.data_handler = data_handler
self.portfolio = portfolio
self.execution_handler = execution_handler
self.position_sizer = position_sizer
self.orders_filters = []
self.monitor = monitor
self.timer = timer
self.order_factory = order_factory
self.time_flow_controller = time_flow_controller
