def _data_array_to_dataframe(self, prices_data_array: QFDataArray,
frequency: Frequency):
"""
Converts a QFDataArray into a DataFrame by removing the "Price Field" axis.
Every index (e.g. 15:00) denotes the close price of the time range beginning at this time (15:00 - 15:01)
The only exception is the time range 1 minute before market open (e.g. 9:29 - 9:30 if market opens 9:30). The
price for this time range, denotes the OPEN price of 9:30 - 9:31.
"""
original_dates = list(prices_data_array.dates.to_index())
dates = prices_data_array.resample(dates='1D').first().dates.to_index()
market_open_datetimes = [
price_datetime + MarketOpenEvent.trigger_time()
for price_datetime in dates if price_datetime +
MarketOpenEvent.trigger_time() in original_dates
]
shifted_open_datetimes = [
price_datetime - frequency.time_delta()
for price_datetime in market_open_datetimes
]
new_dates = list(set(original_dates + shifted_open_datetimes))
new_dates = sorted(new_dates)
prices_df = PricesDataFrame(index=new_dates,
columns=prices_data_array.tickers)
prices_df.loc[shifted_open_datetimes, :] = \
prices_data_array.loc[market_open_datetimes, :, PriceField.Open].values
prices_df.loc[original_dates, :] = prices_data_array.loc[
original_dates, :, PriceField.Close].values
return prices_df
def setUp(self):
BeforeMarketOpenEvent.set_trigger_time({
"hour": 8,
"minute": 0,
"second": 0,
"microsecond": 0
})
MarketOpenEvent.set_trigger_time({
"hour": 13,
"minute": 30,
"second": 0,
"microsecond": 0
})
MarketCloseEvent.set_trigger_time({
"hour": 20,
"minute": 0,
"second": 0,
"microsecond": 0
})
AfterMarketCloseEvent.set_trigger_time({
"hour": 21,
"minute": 0,
"second": 0,
"microsecond": 0
})
def __init__(self, data_provider: GeneralPriceProvider, settings: Settings,
pdf_exporter: PDFExporter, excel_exporter: ExcelExporter):
self._logger = qf_logger.getChild(self.__class__.__name__)
self._backtest_name = "Backtest Results"
self._initial_cash = 10000000
self._initial_risk = None
self._benchmark_tms = None
self._monitor_settings = None
self._contract_ticker_mapper = SimulatedBloombergContractTickerMapper()
self._commission_model_type = FixedCommissionModel
self._commission_model_kwargs = {"commission": 0.0}
self._slippage_model_type = PriceBasedSlippage
self._slippage_model_kwargs = {
"slippage_rate": 0.0,
"max_volume_share_limit": None
}
self._position_sizer_type = SimplePositionSizer
self._position_sizer_kwargs = dict()
self._orders_filter_types_params = [
] # type: List[Tuple[Type[OrdersFilter], Dict]]
self._signals_register = None
self._data_provider = data_provider
self._settings = settings
self._pdf_exporter = pdf_exporter
self._excel_exporter = excel_exporter
self._frequency = None
self._scheduling_time_delay = RelativeDelta(minutes=1)
BeforeMarketOpenEvent.set_trigger_time({
"hour": 8,
"minute": 0,
"second": 0,
"microsecond": 0
})
MarketOpenEvent.set_trigger_time({
"hour": 13,
"minute": 30,
"second": 0,
"microsecond": 0
})
MarketCloseEvent.set_trigger_time({
"hour": 20,
"minute": 0,
"second": 0,
"microsecond": 0
})
AfterMarketCloseEvent.set_trigger_time({
"hour": 23,
"minute": 00,
"second": 0,
"microsecond": 0
})
def test_get_last_price_single_ticker(self):
# just test if getting single ticker value works, when a single ticker (not wrapped in a list) is passed
self.timer.set_current_time(
str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() +
RelativeDelta(hours=1))
single_price = self.data_handler.get_last_available_price(
self.spx_index_ticker)
self.assertTrue(isinstance(single_price, float))
# at market open
self.timer.set_current_time(
str_to_date("2018-01-31") + MarketOpenEvent.trigger_time())
at_market_open = self.data_handler.get_last_available_price(
[self.spx_index_ticker])
self.assertEqual(self.spx_index_ticker, at_market_open.index[0])
self.assertEqual(single_price, at_market_open[0])
# during the trading session
self.timer.set_current_time(
str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() +
RelativeDelta(hours=1))
during_the_day_last_prices = self.data_handler.get_last_available_price(
[self.spx_index_ticker])
self.assertEqual(self.spx_index_ticker,
during_the_day_last_prices.index[0])
self.assertEqual(single_price, during_the_day_last_prices[0])
# after the trading session
self.timer.set_current_time(
str_to_date("2018-01-31") + MarketCloseEvent.trigger_time() +
RelativeDelta(hours=1))
after_close_last_prices = self.data_handler.get_last_available_price(
[self.spx_index_ticker])
self.assertEqual(self.spx_index_ticker,
after_close_last_prices.index[0])
self.assertNotEqual(during_the_day_last_prices[0],
after_close_last_prices[0])
# before the trading session
self.timer.set_current_time(
str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() -
RelativeDelta(hours=1))
before_trading_session_prices = self.data_handler.get_last_available_price(
[self.spx_index_ticker])
self.assertEqual(self.spx_index_ticker,
before_trading_session_prices.index[0])
self.assertNotEqual(during_the_day_last_prices[0],
before_trading_session_prices[0])
self.assertNotEqual(after_close_last_prices[0],
before_trading_session_prices[0])
def setUpClass(cls):
cls.YESTERDAY_OPEN = str_to_date("2018-01-29 13:30:00.000000", DateFormat.FULL_ISO)
cls.YESTERDAY_CLOSE = str_to_date("2018-01-29 20:00:00.000000", DateFormat.FULL_ISO)
cls.TODAY_BEFORE_OPEN = str_to_date("2018-01-30 06:00:00.000000", DateFormat.FULL_ISO)
cls.TODAY_OPEN = str_to_date("2018-01-30 13:30:00.000000", DateFormat.FULL_ISO)
cls.TODAY_MIDDLE_DAY = str_to_date("2018-01-30 15:00:00.000000", DateFormat.FULL_ISO)
cls.TODAY_CLOSE = str_to_date("2018-01-30 20:00:00.000000", DateFormat.FULL_ISO)
cls.TODAY_AFTER_CLOSE = str_to_date("2018-01-30 20:00:00.000000", DateFormat.FULL_ISO)
MarketOpenEvent.set_trigger_time({"hour": 13, "minute": 30, "second": 0, "microsecond": 0})
MarketCloseEvent.set_trigger_time({"hour": 20, "minute": 0, "second": 0, "microsecond": 0})
def setUp(self):
self.timer, self.tickers, self.data_handler = _get_test_case_set_up()
MarketOpenEvent.set_trigger_time({
"hour": 13,
"minute": 30,
"second": 0,
"microsecond": 0
})
MarketCloseEvent.set_trigger_time({
"hour": 20,
"minute": 0,
"second": 0,
"microsecond": 0
})
def test_get_open_price_when_end_date_is_today_before_market_close__single_ticker(
self):
self.timer.set_current_time(
str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() +
RelativeDelta(hours=1))
current_date = datetime(self.timer.now().year,
self.timer.now().month,
self.timer.now().day)
open_prices_tms = self.data_handler.get_price(self.spx_index_ticker,
PriceField.Open,
self.start_date,
self.timer.now())
self.assertEqual(self.start_date,
open_prices_tms.index[0].to_pydatetime())
self.assertEqual(current_date,
open_prices_tms.index[-1].to_pydatetime())
prices_df = self.data_handler.get_price(
self.spx_index_ticker, [PriceField.Open, PriceField.Close],
self.start_date, self.timer.now())
self.assertEqual(self.start_date, prices_df.index[0].to_pydatetime())
self.assertEqual(current_date, prices_df.index[-1].to_pydatetime())
last_bar = prices_df.iloc[-1]
self.assertTrue(isnan(last_bar.loc[PriceField.Close]))
self.assertIsNotNone(last_bar.loc[PriceField.Open])
def test_historical_price_many_tickers_one_field(self):
self.timer.set_current_time(
str_to_date("2018-01-04") + MarketOpenEvent.trigger_time() +
RelativeDelta(hours=1))
result_df = self.data_handler.historical_price([self.microsoft_ticker],
PriceField.Open,
nr_of_bars=5)
self.assertEqual(PricesDataFrame, type(result_df))
expected_dates_idx = pd.DatetimeIndex([
'2017-12-27', '2017-12-28', '2017-12-29', '2018-01-02',
'2018-01-03'
],
name=DATES)
assert_same_index(expected_dates_idx,
result_df.index,
check_index_type=True,
check_names=True)
expected_tickers_idx = pd.Index([self.microsoft_ticker], name=TICKERS)
assert_same_index(expected_tickers_idx,
result_df.columns,
check_index_type=True,
check_names=True)
def _get_prices_df(self, ticker: Ticker, start_date: datetime, end_date: datetime) -> PricesDataFrame:
""" Returns non-adjusted open and close prices, indexed with the Market Open and Market Close time."""
if isinstance(ticker, FutureTicker):
ticker.initialize_data_provider(SettableTimer(end_date), self._data_provider)
tickers_chain = ticker.get_expiration_dates()
if start_date >= tickers_chain.index[-1] or end_date <= tickers_chain.index[0]:
# If the futures chain starts after the _end_date or ends before the _start_date - no data available
return PricesDataFrame()
# Get all tickers from the chain that were valid between the start_date and expiration date of the
# currently valid ticker
end_date = tickers_chain[tickers_chain == ticker.get_current_specific_ticker()].index[0]
tickers_chain = tickers_chain.loc[start_date:end_date]
tickers = tickers_chain.values.tolist()
open_prices = self._data_provider.get_price(tickers, PriceField.Open, start_date, end_date)
close_prices = self._data_provider.get_price(tickers, PriceField.Close, start_date, end_date)
else:
open_prices = self._data_provider.get_price([ticker], PriceField.Open, start_date, end_date)
close_prices = self._data_provider.get_price([ticker], PriceField.Close, start_date, end_date)
open_prices.index = [dt + MarketOpenEvent.trigger_time() for dt in open_prices.index]
close_prices.index = [dt + MarketCloseEvent.trigger_time() for dt in close_prices.index]
prices = concat([open_prices, close_prices]).sort_index()
return prices
def setUp(self):
self.timer, self.tickers, self.data_handler = _get_test_case_set_up()
self.tickers_index = pd.Index(self.tickers, name='tickers')
self.fields_index = pd.Index(PriceField.ohlcv(), name='fields')
MarketOpenEvent.set_trigger_time({
"hour": 13,
"minute": 30,
"second": 0,
"microsecond": 0
})
MarketCloseEvent.set_trigger_time({
"hour": 20,
"minute": 0,
"second": 0,
"microsecond": 0
})
def test_no_orders_executed_on_market_open(self):
# Move before the market open event
self._set_bar_for_today(open_price=105.0,
high_price=110.0,
low_price=100.0,
close_price=105.0,
volume=100000000.0)
self.timer.set_current_time(self.timer.now() +
MarketOpenEvent.trigger_time() -
RelativeDelta(
minutes=self.number_of_minutes))
self.exec_handler.assign_order_ids([
self.stop_loss_order_1, self.stop_loss_order_2,
self.stop_loss_order_3
])
# Trigger the order execution event (the function also forwards time into the future)
self._trigger_single_time_event()
self.exec_handler.on_market_open(...)
self.portfolio.transact_transaction.assert_not_called()
self.monitor.record_transaction.assert_not_called()
actual_orders = self.exec_handler.get_open_orders()
expected_orders = [
self.stop_loss_order_1, self.stop_loss_order_2,
self.stop_loss_order_3
]
assert_lists_equal(expected_orders, actual_orders)
def _data_array_to_dataframe(self, prices_data_array: QFDataArray):
"""
Converts a QFDataArray into a DataFrame by removing the "Price Field" axis.
In order to remove it open and close prices get different time component in their corresponding datetimes
(open prices will get the time of `MarketOpenEvent` and close prices will get the time of `MarketCloseEvent`).
"""
original_dates = prices_data_array.dates.to_index()
market_open_datetimes = [
price_datetime + MarketOpenEvent.trigger_time()
for price_datetime in original_dates
]
market_close_datetimes = [
price_datetime + MarketCloseEvent.trigger_time()
for price_datetime in original_dates
]
new_dates = set(market_open_datetimes + market_close_datetimes)
prices_df = PricesDataFrame(index=new_dates,
columns=prices_data_array.tickers)
prices_df.loc[
market_open_datetimes, :] = prices_data_array.loc[:, :, PriceField.
Open].values
prices_df.loc[
market_close_datetimes, :] = prices_data_array.loc[:, :,
PriceField.
Close].values
prices_df.sort_index(inplace=True)
return prices_df
def test_get_open_price_when_end_date_is_today_before_market_close__multiple_tickers(
self):
self.timer.set_current_time(
str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() +
RelativeDelta(hours=1))
current_date = datetime(self.timer.now().year,
self.timer.now().month,
self.timer.now().day)
tickers = [self.spx_index_ticker, self.microsoft_ticker]
open_prices_tms = self.data_handler.get_price(tickers, PriceField.Open,
self.start_date,
self.timer.now())
self.assertEqual(self.start_date,
open_prices_tms.index[0].to_pydatetime())
self.assertEqual(current_date,
open_prices_tms.index[-1].to_pydatetime())
prices_data_array = self.data_handler.get_price(
tickers, [PriceField.Open, PriceField.Close], self.start_date,
self.timer.now())
last_bar_df = prices_data_array.loc[current_date, :, :].to_pandas()
self.assertTrue(last_bar_df[PriceField.Close].isna().all())
self.assertTrue(last_bar_df[PriceField.Open].notna().all())
def test_market_open_rule(self):
market_open_event = MarketOpenEvent()
self.timer.set_current_time(str_to_date("2018-01-01 00:00:00.000000", DateFormat.FULL_ISO))
now = self.timer.now()
next_trigger_time = market_open_event.next_trigger_time(now)
self.assertEqual(str_to_date("2018-01-01 13:30:00.000000", DateFormat.FULL_ISO), next_trigger_time)
self.timer.set_current_time(str_to_date("2018-01-01 09:29:59.999999", DateFormat.FULL_ISO))
now = self.timer.now()
next_trigger_time = market_open_event.next_trigger_time(now)
self.assertEqual(str_to_date("2018-01-01 13:30:00.000000", DateFormat.FULL_ISO), next_trigger_time)
self.timer.set_current_time(str_to_date("2018-01-01 13:30:00.000000", DateFormat.FULL_ISO))
now = self.timer.now()
next_trigger_time = market_open_event.next_trigger_time(now)
self.assertEqual(str_to_date("2018-01-02 13:30:00.000000", DateFormat.FULL_ISO), next_trigger_time)
def __init__(self,
tickers: Union[Ticker, Sequence[Ticker]],
start_date: datetime,
end_date: datetime,
data_handler: DataHandler,
alpha_models: Union[AlphaModel, Sequence[AlphaModel]],
settings: Settings,
pdf_exporter: PDFExporter,
only_entry_signals: bool = True,
title: str = "Signals Plotter"):
super().__init__(settings, pdf_exporter, title)
# Set the market open and close events in order to use the data handler (necessary to e.g. compute the market
# close time of the previous day)
MarketOpenEvent.set_trigger_time({
"hour": 13,
"minute": 30,
"second": 0,
"microsecond": 0
})
MarketCloseEvent.set_trigger_time({
"hour": 20,
"minute": 0,
"second": 0,
"microsecond": 0
})
self.tickers, _ = convert_to_list(tickers, Ticker)
self.alpha_models, _ = convert_to_list(alpha_models, AlphaModel)
self.start_date = start_date
self.end_date = end_date
self.data_handler = data_handler
assert isinstance(self.data_handler.timer, SettableTimer)
self.timer: SettableTimer = self.data_handler.timer
self.only_entry_signals = only_entry_signals
for ticker in tickers:
if isinstance(ticker, FutureTicker):
ticker.initialize_data_provider(
self.timer, self.data_handler.data_provider)
def _make_mock_data_array(self, tickers, fields):
all_dates_market_open = pd.date_range(
start=self.data_start_date + MarketOpenEvent.trigger_time(),
end=self.data_end_date + MarketOpenEvent.trigger_time(),
freq="B")
all_dates_market_close = pd.date_range(
start=self.data_start_date + MarketCloseEvent.trigger_time() -
Frequency.MIN_1.time_delta(),
end=self.data_end_date + MarketCloseEvent.trigger_time() -
Frequency.MIN_1.time_delta(),
freq="B")
num_of_dates = len(all_dates_market_open)
num_of_tickers = len(tickers)
num_of_fields = len(fields)
start_value = 100.0
values = np.arange(
start_value,
num_of_dates * num_of_tickers * num_of_fields + start_value)
reshaped_values = np.reshape(
values, (num_of_dates, num_of_tickers, num_of_fields))
mocked_result_market_open = QFDataArray.create(all_dates_market_open,
tickers,
fields,
data=reshaped_values)
mocked_result_market_close = QFDataArray.create(all_dates_market_close,
tickers,
fields,
data=reshaped_values)
mocked_result_market_close.loc[:, :, PriceField.Low] -= 5.0
mocked_result_market_close.loc[:, :, PriceField.High] += 5.0
all_dates = all_dates_market_open.union(all_dates_market_close)
mocked_result = QFDataArray.create(all_dates, tickers, fields)
mocked_result.loc[
all_dates_market_open, :, :] = mocked_result_market_open.loc[:, :, :]
mocked_result.loc[
all_dates_market_close, :, :] = mocked_result_market_close.loc[:, :, :]
self._add_test_cases(mocked_result, tickers)
return mocked_result
def test_get_last_price_with_multiple_tickers_when_current_data_is_unavailable(
self):
self.timer.set_current_time(
str_to_date("2018-01-01") + MarketOpenEvent.trigger_time() +
RelativeDelta(hours=1))
last_prices = self.data_handler.get_last_available_price(
[self.spx_index_ticker, self.google_ticker])
self.assertEqual(self.spx_index_ticker, last_prices.index[0])
self.assertEqual(self.google_ticker, last_prices.index[1])
def _get_current_prices(self, tickers: Sequence[Ticker]):
"""
Function used to obtain the current prices for the tickers in order to further calculate fill prices for orders.
The function uses data provider and not data handler, as it is necessary to get the current bar at each point
in time to compute the fill prices.
"""
if not tickers:
return QFSeries()
assert self._frequency >= Frequency.DAILY, "Lower than daily frequency is not supported by the simulated" \
" executor"
# Compute the time ranges, used further by the get_price function
current_datetime = self._timer.now()
market_close_time = current_datetime + MarketCloseEvent.trigger_time(
) == current_datetime
market_open_time = current_datetime + MarketOpenEvent.trigger_time(
) == current_datetime
# In case of daily frequency, current price may be returned only at the Market Open or Market Close time
if self._frequency == Frequency.DAILY and not (market_open_time
or market_close_time):
return QFSeries(index=tickers)
if self._frequency == Frequency.DAILY:
# Remove the time part from the datetime in case of daily frequency
current_datetime = date_to_datetime(current_datetime.date())
start_time_range = current_datetime - self._frequency.time_delta()
end_time_range = current_datetime
elif market_close_time:
# At the market close, in order to get the current price we need to take a bar that ends at the current time
# and use the close price value
start_time_range = current_datetime - self._frequency.time_delta()
end_time_range = current_datetime
current_datetime = start_time_range
else:
# At any other time during the day, in order to get the current price we need to take the bar that starts at
# the current time and use the open price value
start_time_range = current_datetime
end_time_range = current_datetime + self._frequency.time_delta()
price_field = PriceField.Close if market_close_time else PriceField.Open
prices_df = self._data_provider.get_price(tickers, price_field,
start_time_range,
end_time_range,
self._frequency)
try:
prices_series = prices_df.loc[current_datetime]
except KeyError:
prices_series = QFSeries(index=tickers)
prices_series.name = "Current prices series"
return prices_series
def test_get_price_aggregation_single_ticker(self):
MarketOpenEvent.set_trigger_time({"hour": 17, "minute": 13, "second": 0, "microsecond": 0})
dp = PortaraDataProvider(self.futures_path, self.ticker, PriceField.Close, self.start_date, self.end_date,
Frequency.MIN_1)
prices = dp.get_price(self.ticker, PriceField.Close, self.start_date, self.end_date, Frequency.MIN_1)
prices5 = dp.get_price(self.ticker, PriceField.Close, self.start_date, self.end_date, Frequency.MIN_5)
prices15 = dp.get_price(self.ticker, PriceField.Close, self.start_date, self.end_date, Frequency.MIN_15)
self.assertTrue(len(prices5))
self.assertEqual(type(prices5), PricesSeries)
self.assertEqual(Frequency.infer_freq(prices5.index), Frequency.MIN_5)
assert_series_equal(prices5, prices.resample('5T', origin=datetime(2021, 6, 17, 17, 13)).last().dropna(),
check_names=False)
self.assertTrue(len(prices15))
self.assertEqual(type(prices15), PricesSeries)
self.assertEqual(Frequency.infer_freq(prices15.index), Frequency.MIN_15)
assert_series_equal(prices15, prices.resample('15T', origin=datetime(2021, 6, 17, 17, 13)).last().dropna(),
check_names=False)
def test_get_history_when_end_date_is_today_before_market_close(self):
self.timer.set_current_time(
str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() +
RelativeDelta(hours=1))
prices_tms = self.data_handler.get_history(self.spx_index_ticker,
self.get_history_field,
self.start_date,
self.end_date)
self.assertEqual(self.start_date, prices_tms.index[0].to_pydatetime())
self.assertEqual(self.end_date_trimmed,
prices_tms.index[-1].to_pydatetime())
def setUp(self):
MarketOpenEvent.set_trigger_time({"hour": 13, "minute": 30, "second": 0, "microsecond": 0})
MarketCloseEvent.set_trigger_time({"hour": 20, "minute": 0, "second": 0, "microsecond": 0})
self.start_date = str_to_date("2018-02-04")
self.number_of_minutes = 5
before_close = self.start_date + MarketCloseEvent.trigger_time() - RelativeDelta(minutes=self.number_of_minutes)
self.msft_ticker = BloombergTicker(self.MSFT_TICKER_STR)
self.timer = SettableTimer(initial_time=before_close)
self.data_handler = Mock(spec=DataHandler)
self.data_handler.data_provider = Mock(spec=DataProvider)
scheduler = Mock(spec=Scheduler)
ScheduleOrderExecutionEvent.clear()
# Set the periodic bar events to intraday trading
IntradayBarEvent.frequency = Frequency.MIN_1
commission_model = FixedCommissionModel(commission=0.0)
self.monitor = Mock(spec=AbstractMonitor)
self.portfolio = Mock(spec=Portfolio)
slippage_model = PriceBasedSlippage(0.0, self.data_handler)
self.exec_handler = SimulatedExecutionHandler(self.data_handler, self.timer, scheduler, self.monitor,
commission_model, self.portfolio, slippage_model,
RelativeDelta(minutes=self.number_of_minutes))
self._set_last_available_price(100.0)
self.stop_loss_order_1 = Order(self.msft_ticker, quantity=-1, execution_style=StopOrder(95.0),
time_in_force=TimeInForce.GTC)
self.stop_loss_order_2 = Order(self.msft_ticker, quantity=-1, execution_style=StopOrder(90.0),
time_in_force=TimeInForce.GTC)
self.stop_loss_order_3 = Order(self.msft_ticker, quantity=-1, execution_style=StopOrder(50.0),
time_in_force=TimeInForce.DAY)
self.exec_handler.assign_order_ids([self.stop_loss_order_1, self.stop_loss_order_2, self.stop_loss_order_3])
def setUp(self):
try:
self.price_data_provider = get_data_provider()
except Exception as e:
raise self.skipTest(e)
self.timer = SettableTimer()
self.data_handler = DailyDataHandler(self.price_data_provider,
self.timer)
MarketOpenEvent.set_trigger_time({
"hour": 13,
"minute": 30,
"second": 0,
"microsecond": 0
})
MarketCloseEvent.set_trigger_time({
"hour": 20,
"minute": 0,
"second": 0,
"microsecond": 0
})
def setUp(self):
self.timer = SettableTimer()
self.tickers = [
QuandlTicker("MSFT", "WIKI"),
QuandlTicker("AAPL", "WIKI")
]
price_data_provider_mock = self._create_price_provider_mock(
self.tickers)
self.data_handler = DailyDataHandler(price_data_provider_mock,
self.timer)
MarketOpenEvent.set_trigger_time({
"hour": 13,
"minute": 30,
"second": 0,
"microsecond": 0
})
MarketCloseEvent.set_trigger_time({
"hour": 20,
"minute": 0,
"second": 0,
"microsecond": 0
})
def test_get_history_with_multiple_tickers(self):
self.timer.set_current_time(
str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() +
RelativeDelta(hours=1))
resilt_df = self.data_handler.get_history(
[self.microsoft_ticker, self.google_ticker],
self.get_history_field, self.start_date, self.end_date_trimmed)
self.assertEqual(self.microsoft_ticker, resilt_df.columns[0])
self.assertEqual(self.google_ticker, resilt_df.columns[1])
self.assertEqual(self.start_date, resilt_df.index[0].to_pydatetime())
self.assertEqual(self.end_date_trimmed,
resilt_df.index[-1].to_pydatetime())
self.assertEqual(resilt_df.shape, (20, 2))
def _filter_transactions(self, ticker: Ticker, transactions: List[Transaction], start_date: datetime,
end_date: datetime) -> QFSeries:
""" Filters out transactions, which do not correspond to the given ticker and returns a QFSeries of remaining
transactions. Only transactions between start_date market open and end_date market close are considered. """
transactions = [t for t in transactions if start_date + MarketOpenEvent.trigger_time()
<= t.time <= end_date + MarketCloseEvent.trigger_time()]
if isinstance(ticker, FutureTicker):
transactions_for_tickers = [t for t in transactions if ticker.belongs_to_family(t.ticker)]
else:
transactions_for_tickers = [t for t in transactions if ticker == t.ticker]
transactions_records = [(t, t.time) for t in transactions_for_tickers]
transactions_series = QFDataFrame.from_records(transactions_records, columns=["Transaction", "Index"]) \
.set_index("Index").iloc[:, 0]
return transactions_series
def _aggregate_intraday_data(self, data_array, start_date: datetime,
end_date: datetime, tickers: Sequence[Ticker],
fields, frequency: Frequency):
"""
Function, which aggregates the intraday data array for various dates and returns a new data array with data
sampled with the given frequency.
"""
# If the data is of intraday data type, which spans over multiple days, the base parameter of resample()
# function should be adjusted differently for the first day.
#
# Therefore, the data array is divided into two separate arrays data_array_1, data_array_2 - first containing
# only the first day, and the second one - containing all other dates.
end_of_day = start_date + RelativeDelta(hour=23, minute=59, second=59)
_end_date = end_of_day if (end_of_day < end_date) else end_date
# Get both parts of the data array
data_array_1 = data_array.loc[start_date:_end_date, :, :]
data_array_2 = data_array.loc[end_of_day:end_date, :, :]
if len(data_array_1) > 0:
base_data_array_1 = pd.to_datetime(
data_array_1[DATES].values[0]).minute
data_array_1 = data_array_1.resample(
dates=frequency.to_pandas_freq(),
base=base_data_array_1,
label='left',
skipna=True).apply(
lambda x: self._aggregate_data_array(x, tickers, fields))
if len(data_array_2) > 0:
base_data_array_2 = MarketOpenEvent.trigger_time().minute
data_array_2 = data_array_2.resample(
dates=frequency.to_pandas_freq(),
base=base_data_array_2,
label='left',
skipna=True).apply(
lambda x: self._aggregate_data_array(x, tickers, fields))
data_array = QFDataArray.concat([data_array_1, data_array_2],
dim=DATES)
return data_array
def test_historical_price_many_tickers_many_fields(self):
self.timer.set_current_time(
str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() +
RelativeDelta(hours=1))
result_array = self.data_handler.historical_price(
[self.microsoft_ticker], [PriceField.Open, PriceField.Close],
nr_of_bars=5)
self.assertEqual(QFDataArray, type(result_array))
self.assertEqual((5, 1, 2), result_array.shape)
expected_dates_str = [
"2018-01-24", "2018-01-25", "2018-01-26", "2018-01-29",
"2018-01-30"
]
expected_dates = [
str_to_date(date_str) for date_str in expected_dates_str
]
assert_same_index(pd.DatetimeIndex(expected_dates, name=DATES),
result_array.dates.to_index(),
check_index_type=True,
check_names=True)
class TestFastAlphaModelsTester(TestCase):
apple_ticker = QuandlTicker("AAPL", "WIKI")
ibm_ticker = QuandlTicker("IBM", "WIKI")
tickers = [apple_ticker, ibm_ticker]
test_start_date = str_to_date("2015-01-01")
test_end_date = str_to_date("2015-01-31")
data_start_date = str_to_date("2014-12-25")
data_end_date = test_end_date
frequency = Frequency.DAILY
MarketOpenEvent.set_trigger_time({
"hour": 13,
"minute": 30,
"second": 0,
"microsecond": 0
})
MarketCloseEvent.set_trigger_time({
"hour": 20,
"minute": 0,
"second": 0,
"microsecond": 0
})
def setUp(self):
all_fields = PriceField.ohlcv()
self._mocked_prices_arr = self._make_mock_data_array(
self.tickers, all_fields)
self._price_provider_mock = PresetDataProvider(self._mocked_prices_arr,
self.data_start_date,
self.data_end_date,
self.frequency)
self.timer = SettableTimer()
self.contract_ticker_mapper = DummyQuandlContractTickerMapper()
self.alpha_model_type = DummyAlphaModel
@classmethod
def _make_mock_data_array(cls, tickers, fields):
all_dates_index = pd.bdate_range(start=cls.data_start_date,
end=cls.test_end_date)
num_of_dates = len(all_dates_index)
num_of_tickers = len(tickers)
num_of_fields = len(fields)
start_value = 100.0
values = np.arange(
start_value,
num_of_dates * num_of_tickers * num_of_fields + start_value)
reshaped_values = np.reshape(
values, (num_of_dates, num_of_tickers, num_of_fields))
mocked_result = QFDataArray.create(all_dates_index,
tickers,
fields,
data=reshaped_values)
mocked_result.loc[:, :, PriceField.Low] -= 50.0
mocked_result.loc[:, :, PriceField.High] += 50.0
return mocked_result
def test_alpha_models_tester(self):
first_param_set = (10, Exposure.LONG)
second_param_set = (5, Exposure.SHORT)
data_handler = FastDataHandler(self._price_provider_mock, self.timer)
params = [
FastAlphaModelTesterConfig(
self.alpha_model_type, {
"period_length": 10,
"first_suggested_exposure": Exposure.LONG,
"risk_estimation_factor": None
}, ("period_length", "first_suggested_exposure")),
FastAlphaModelTesterConfig(
self.alpha_model_type, {
"period_length": 5,
"first_suggested_exposure": Exposure.SHORT,
"risk_estimation_factor": None
}, ("period_length", "first_suggested_exposure"))
]
tester = FastAlphaModelTester(params, self.tickers,
self.contract_ticker_mapper,
self.test_start_date, self.test_end_date,
data_handler, self.timer)
backtest_summary = tester.test_alpha_models()
self.assertEqual(self.tickers, backtest_summary.tickers)
self.assertEqual(DummyAlphaModel, backtest_summary.alpha_model_type)
backtest_summary_elements = backtest_summary.elements_list
self.assertEqual(2 * (len(self.tickers) + 1),
len(backtest_summary_elements))
# check first backtest summary element - trades
first_elem = backtest_summary_elements[2]
self.assertEqual(first_param_set, first_elem.model_parameters)
expected_trades_data = [[
self.apple_ticker,
str_to_date("2015-01-02"),
str_to_date("2015-01-16"), (260.0 / 160.0 - 1), 1.0
],
[
self.ibm_ticker,
str_to_date("2015-01-02"),
str_to_date("2015-01-16"),
(265.0 / 165.0 - 1), 1.0
]]
generated_trades_data = [[
self.contract_ticker_mapper.contract_to_ticker(t.contract),
t.start_time, t.end_time, t.pnl, t.direction
] for t in first_elem.trades]
self.assertCountEqual(generated_trades_data, expected_trades_data)
# check first backtest summary element - returns
all_dates_index = pd.bdate_range(start=self.test_start_date + BDay(2),
end=self.test_end_date)
expected_returns = SimpleReturnsSeries(
index=all_dates_index,
data=[
0.0615530, 0.0579832, 0.0548048, 0.0519568, 0.0493902,
0.0470653, 0.0449495, 0.0430157, 0.0412415, 0.0396078,
0.0000000, 0.0000000, 0.0000000, 0.0000000, 0.0000000,
0.0000000, 0.0000000, 0.0000000, 0.0000000, 0.0000000
])
assert_series_equal(expected_returns, first_elem.returns_tms)
# check second backtest summary element
second_elem = backtest_summary_elements[5]
self.assertEqual(second_param_set, second_elem.model_parameters)
expected_trades_data = [[
self.apple_ticker,
str_to_date("2015-01-02"),
str_to_date("2015-01-09"), (1 - 210.0 / 160.0), -1.0
],
[
self.apple_ticker,
str_to_date("2015-01-16"),
str_to_date("2015-01-23"),
(310.0 / 260.0 - 1), 1.0
],
[
self.apple_ticker,
str_to_date("2015-01-23"),
str_to_date("2015-01-30"),
(1 - 360.0 / 310.0), -1.0
],
[
self.ibm_ticker,
str_to_date("2015-01-02"),
str_to_date("2015-01-09"),
(1 - 215.0 / 165.0), -1.0
],
[
self.ibm_ticker,
str_to_date("2015-01-16"),
str_to_date("2015-01-23"),
(315.0 / 265.0 - 1), 1.0
],
[
self.ibm_ticker,
str_to_date("2015-01-23"),
str_to_date("2015-01-30"),
(1 - 365.0 / 315.0), -1.0
]]
generated_trades_data = [[
self.contract_ticker_mapper.contract_to_ticker(t.contract),
t.start_time, t.end_time, t.pnl, t.direction
] for t in second_elem.trades]
self.assertCountEqual(expected_trades_data, generated_trades_data)
# check second backtest summary element - returns
all_dates_index = pd.bdate_range(start=self.test_start_date + BDay(2),
end=self.test_end_date)
expected_returns = SimpleReturnsSeries(
index=all_dates_index,
data=[
-0.0615530, -0.0579832, -0.0548048, -0.0519568, -0.0493902,
0.0000000, 0.0000000, 0.0000000, 0.0000000, 0.0000000,
0.0380987, 0.0367003, 0.0354010, 0.0341905, 0.0330601,
-0.0320020, -0.0310096, -0.0300769, -0.0291986, -0.0283702
])
assert_series_equal(expected_returns, second_elem.returns_tms)
def test_get_last_price_with_empty_tickers_list(self):
self.timer.set_current_time(
str_to_date("2018-01-31") + MarketOpenEvent.trigger_time() +
RelativeDelta(hours=1))
last_prices = self.data_handler.get_last_available_price([])
assert_series_equal(QFSeries(), last_prices)
class TradingSessionForTests(TradingSession):
"""
Encapsulates the settings and components for carrying out a backtest session. Pulls for data every day.
"""
# initialise market time
BeforeMarketOpenEvent.set_trigger_time({"hour": 7, "minute": 30, "second": 0, "microsecond": 0})
MarketOpenEvent.set_trigger_time({"hour": 13, "minute": 30, "second": 0, "microsecond": 0})
MarketCloseEvent.set_trigger_time({"hour": 20, "minute": 0, "second": 0, "microsecond": 0})
AfterMarketCloseEvent.set_trigger_time({"hour": 21, "minute": 0, "second": 0, "microsecond": 0})
def __init__(self, data_provider: DataProvider, start_date, end_date, initial_cash,
frequency: Frequency = Frequency.MIN_1):
"""
Set up the backtest variables according to what has been passed in.
"""
super().__init__()
self.logger = qf_logger.getChild(self.__class__.__name__)
self.logger.info(
"\n".join([
"Testing the Backtester:",
"Start date: {:s}".format(date_to_str(start_date)),
"End date: {:s}".format(date_to_str(end_date)),
"Initial cash: {:.2f}".format(initial_cash),
"Frequency of the simulated execution handler: {}".format(frequency)
])
)
timer = SettableTimer(start_date)
notifiers = Notifiers(timer)
if frequency <= Frequency.DAILY:
data_handler = DailyDataHandler(data_provider, timer)
else:
data_handler = IntradayDataHandler(data_provider, timer)
contract_ticker_mapper = SimulatedBloombergContractTickerMapper()
portfolio = Portfolio(data_handler, initial_cash, timer, contract_ticker_mapper)
signals_register = BacktestSignalsRegister()
backtest_result = BacktestResult(portfolio=portfolio, backtest_name="Testing the Backtester",
start_date=start_date, end_date=end_date, signals_register=signals_register)
monitor = Mock(spec=BacktestMonitor)
commission_model = FixedCommissionModel(0.0)
slippage_model = PriceBasedSlippage(0.0, data_provider, contract_ticker_mapper)
execution_handler = SimulatedExecutionHandler(
data_handler, timer, notifiers.scheduler, monitor, commission_model,
contract_ticker_mapper, portfolio, slippage_model)
broker = BacktestBroker(portfolio, execution_handler)
order_factory = OrderFactory(broker, data_handler, contract_ticker_mapper)
event_manager = self._create_event_manager(timer, notifiers)
time_flow_controller = BacktestTimeFlowController(
notifiers.scheduler, event_manager, timer, notifiers.empty_queue_event_notifier, end_date
)
position_sizer = SimplePositionSizer(broker, data_handler, order_factory, contract_ticker_mapper,
signals_register)
self.logger.info(
"\n".join([
"Configuration of components:",
"Position sizer: {:s}".format(position_sizer.__class__.__name__),
"Timer: {:s}".format(timer.__class__.__name__),
"Data Provider: {:s}".format(data_provider.__class__.__name__),
"Backtest Result: {:s}".format(backtest_result.__class__.__name__),
"Monitor: {:s}".format(monitor.__class__.__name__),
"Execution Handler: {:s}".format(execution_handler.__class__.__name__),
"Commission Model: {:s}".format(commission_model.__class__.__name__),
"Broker: {:s}".format(broker.__class__.__name__),
"Contract-Ticker Mapper: {:s}".format(contract_ticker_mapper.__class__.__name__)
])
)
self.broker = broker
self.notifiers = notifiers
self.initial_cash = initial_cash
self.start_date = start_date
self.end_date = end_date
self.event_manager = event_manager
self.contract_ticker_mapper = contract_ticker_mapper
self.data_handler = data_handler
self.portfolio = portfolio
self.execution_handler = execution_handler
self.position_sizer = position_sizer
self.orders_filters = []
self.monitor = monitor
self.timer = timer
self.order_factory = order_factory
self.time_flow_controller = time_flow_controller