class Backtest(object):
"""Conducts backtest for strategies trading assets. Assumes we have an input of total returns. Reports historical return statistics
and returns time series.
"""
def __init__(self):
self.logger = LoggerManager().getLogger(__name__)
self._pnl = None
self._portfolio = None
return
def calculate_diagnostic_trading_PnL(self, asset_a_df, signal_df, further_df = [], further_df_labels = []):
"""Calculates P&L table which can be used for debugging purposes,
The table is populated with asset, signal and further dataframes provided by the user, can be used to check signalling methodology.
It does not apply parameters such as transaction costs, vol adjusment and so on.
Parameters
----------
asset_a_df : DataFrame
Asset prices
signal_df : DataFrame
Trade signals (typically +1, -1, 0 etc)
further_df : DataFrame
Further dataframes user wishes to output in the diagnostic output (typically inputs for the signals)
further_df_labels
Labels to append to the further dataframes
Returns
-------
DataFrame with asset, trading signals and returns of the trading strategy for diagnostic purposes
"""
calculations = Calculations()
asset_rets_df = calculations.calculate_returns(asset_a_df)
strategy_rets = calculations.calculate_signal_returns(signal_df, asset_rets_df)
reset_points = ((signal_df - signal_df.shift(1)).abs())
asset_a_df_entry = asset_a_df.copy(deep=True)
asset_a_df_entry[reset_points == 0] = numpy.nan
asset_a_df_entry = asset_a_df_entry.ffill()
asset_a_df_entry.columns = [x + '_entry' for x in asset_a_df_entry.columns]
asset_rets_df.columns = [x + '_asset_rets' for x in asset_rets_df.columns]
strategy_rets.columns = [x + '_strat_rets' for x in strategy_rets.columns]
signal_df.columns = [x + '_final_signal' for x in signal_df.columns]
for i in range(0, len(further_df)):
further_df[i].columns = [x + '_' + further_df_labels[i] for x in further_df[i].columns]
flatten_df =[asset_a_df, asset_a_df_entry, asset_rets_df, strategy_rets, signal_df]
for f in further_df:
flatten_df.append(f)
return calculations.pandas_outer_join(flatten_df)
def calculate_trading_PnL(self, br, asset_a_df, signal_df, contract_value_df = None):
"""Calculates P&L of a trading strategy and statistics to be retrieved later
Calculates the P&L for each asset/signal combination and also for the finally strategy applying appropriate
weighting in the portfolio, depending on predefined parameters, for example:
static weighting for each asset
static weighting for each asset + vol weighting for each asset
static weighting for each asset + vol weighting for each asset + vol weighting for the portfolio
Parameters
----------
br : BacktestRequest
Parameters for the backtest specifying start date, finish data, transaction costs etc.
asset_a_df : pandas.DataFrame
Asset prices to be traded
signal_df : pandas.DataFrame
Signals for the trading strategy
contract_value_df : pandas.DataFrame
Daily size of contracts
"""
calculations = Calculations()
# make sure the dates of both traded asset and signal are aligned properly
asset_df, signal_df = asset_a_df.align(signal_df, join='left', axis = 'index')
if (contract_value_df is not None):
asset_df, contract_value_df = asset_df.align(contract_value_df, join='left', axis='index')
contract_value_df = contract_value_df.fillna(method='ffill') # fill down asset holidays (we won't trade on these days)
# non-trading days
non_trading_days = numpy.isnan(asset_df.values)
# only allow signals to change on the days when we can trade assets
signal_df = signal_df.mask(non_trading_days) # fill asset holidays with NaN signals
signal_df = signal_df.fillna(method='ffill') # fill these down
tc = br.spot_tc_bp
signal_cols = signal_df.columns.values
asset_df_cols = asset_df.columns.values
pnl_cols = []
for i in range(0, len(asset_df_cols)):
pnl_cols.append(asset_df_cols[i] + " / " + signal_cols[i])
asset_df_copy = asset_df.copy()
asset_df = asset_df.fillna(method='ffill') # fill down asset holidays (we won't trade on these days)
returns_df = calculations.calculate_returns(asset_df)
# apply a stop loss/take profit to every trade if this has been specified
# do this before we start to do vol weighting etc.
if hasattr(br, 'take_profit') and hasattr(br, 'stop_loss'):
returns_df = calculations.calculate_returns(asset_df)
temp_strategy_rets_df = calculations.calculate_signal_returns(signal_df, returns_df)
trade_rets_df = calculations.calculate_cum_rets_trades(signal_df, temp_strategy_rets_df)
# pre_signal_df = signal_df.copy()
signal_df = calculations.calculate_risk_stop_signals(signal_df, trade_rets_df, br.stop_loss, br.take_profit)
# make sure we can't trade where asset price is undefined and carry over signal
signal_df = signal_df.mask(non_trading_days) # fill asset holidays with NaN signals
signal_df = signal_df.fillna(method='ffill') # fill these down (when asset is not trading
# signal_df.columns = [x + '_final_signal' for x in signal_df.columns]
# for debugging purposes
# if False:
# signal_df_copy = signal_df.copy()
# trade_rets_df_copy = trade_rets_df.copy()
#
# asset_df_copy.columns = [x + '_asset' for x in temp_strategy_rets_df.columns]
# temp_strategy_rets_df.columns = [x + '_strategy_rets' for x in temp_strategy_rets_df.columns]
# signal_df_copy.columns = [x + '_final_signal' for x in signal_df_copy.columns]
# trade_rets_df_copy.columns = [x + '_cum_trade' for x in trade_rets_df_copy.columns]
#
# to_plot = calculations.pandas_outer_join([asset_df_copy, pre_signal_df, signal_df_copy, trade_rets_df_copy, temp_strategy_rets_df])
# to_plot.to_csv('test.csv')
# do we have a vol target for individual signals?
if hasattr(br, 'signal_vol_adjust'):
if br.signal_vol_adjust is True:
risk_engine = RiskEngine()
if not(hasattr(br, 'signal_vol_resample_type')):
br.signal_vol_resample_type = 'mean'
if not(hasattr(br, 'signal_vol_resample_freq')):
br.signal_vol_resample_freq = None
if not(hasattr(br, 'signal_vol_period_shift')):
br.signal_vol_period_shift = 0
leverage_df = risk_engine.calculate_leverage_factor(returns_df, br.signal_vol_target, br.signal_vol_max_leverage,
br.signal_vol_periods, br.signal_vol_obs_in_year,
br.signal_vol_rebalance_freq, br.signal_vol_resample_freq,
br.signal_vol_resample_type, period_shift=br.signal_vol_period_shift)
signal_df = pandas.DataFrame(
signal_df.values * leverage_df.values, index = signal_df.index, columns = signal_df.columns)
self._individual_leverage = leverage_df # contains leverage of individual signal (before portfolio vol target)
_pnl = calculations.calculate_signal_returns_with_tc_matrix(signal_df, returns_df, tc = tc)
_pnl.columns = pnl_cols
adjusted_weights_matrix = None
# portfolio is average of the underlying signals: should we sum them or average them?
if hasattr(br, 'portfolio_combination'):
if br.portfolio_combination == 'sum':
portfolio = pandas.DataFrame(data = _pnl.sum(axis = 1), index = _pnl.index, columns = ['Portfolio'])
elif br.portfolio_combination == 'mean':
portfolio = pandas.DataFrame(data = _pnl.mean(axis = 1), index = _pnl.index, columns = ['Portfolio'])
adjusted_weights_matrix = self.create_portfolio_weights(br, _pnl, method='mean')
elif isinstance(br.portfolio_combination, dict):
# get the weights for each asset
adjusted_weights_matrix = self.create_portfolio_weights(br, _pnl, method='weighted')
portfolio = pandas.DataFrame(data=(_pnl.values * adjusted_weights_matrix), index=_pnl.index)
is_all_na = pandas.isnull(portfolio).all(axis=1)
portfolio = pandas.DataFrame(portfolio.sum(axis = 1), columns = ['Portfolio'])
# overwrite days when every asset PnL was null is NaN with nan
portfolio[is_all_na] = numpy.nan
else:
portfolio = pandas.DataFrame(data = _pnl.mean(axis = 1), index = _pnl.index, columns = ['Portfolio'])
adjusted_weights_matrix = self.create_portfolio_weights(br, _pnl, method='mean')
portfolio_leverage_df = pandas.DataFrame(data = numpy.ones(len(_pnl.index)), index = _pnl.index, columns = ['Portfolio'])
# should we apply vol target on a portfolio level basis?
if hasattr(br, 'portfolio_vol_adjust'):
if br.portfolio_vol_adjust is True:
risk_engine = RiskEngine()
portfolio, portfolio_leverage_df = risk_engine.calculate_vol_adjusted_returns(portfolio, br = br)
self._portfolio = portfolio
self._signal = signal_df # individual signals (before portfolio leverage)
self._portfolio_leverage = portfolio_leverage_df # leverage on portfolio
# multiply portfolio leverage * individual signals to get final position signals
length_cols = len(signal_df.columns)
leverage_matrix = numpy.repeat(portfolio_leverage_df.values.flatten()[numpy.newaxis,:], length_cols, 0)
# final portfolio signals (including signal & portfolio leverage)
self._portfolio_signal = pandas.DataFrame(
data = numpy.multiply(numpy.transpose(leverage_matrix), signal_df.values),
index = signal_df.index, columns = signal_df.columns)
if hasattr(br, 'portfolio_combination'):
if br.portfolio_combination == 'sum':
pass
elif br.portfolio_combination == 'mean' or isinstance(br.portfolio_combination, dict):
self._portfolio_signal = pandas.DataFrame(data=(self._portfolio_signal.values * adjusted_weights_matrix),
index=self._portfolio_signal.index,
columns=self._portfolio_signal.columns)
else:
self._portfolio_signal = pandas.DataFrame(data=(self._portfolio_signal.values * adjusted_weights_matrix),
index=self._portfolio_signal.index,
columns=self._portfolio_signal.columns)
# calculate each period of trades
self._portfolio_trade = self._portfolio_signal - self._portfolio_signal.shift(1)
self._portfolio_signal_notional = None
self._portfolio_signal_trade_notional = None
self._portfolio_signal_contracts = None
self._portfolio_signal_trade_contracts = None
# also create other measures of portfolio
# portfolio & trades in terms of a predefined notional (in USD)
# portfolio & trades in terms of contract sizes (particularly useful for futures)
if hasattr(br, 'portfolio_notional_size'):
# express positions in terms of the notional size specified
self._portfolio_signal_notional = self._portfolio_signal * br.portfolio_notional_size
self._portfolio_signal_trade_notional = self._portfolio_signal_notional - self._portfolio_signal_notional.shift(1)
# get the positions in terms of the contract sizes
notional_copy = self._portfolio_signal_notional.copy(deep=True)
notional_copy_cols = [x.split('.')[0] for x in notional_copy.columns]
notional_copy_cols = [x + '.contract-value' for x in notional_copy_cols]
notional_copy.columns = notional_copy_cols
contract_value_df = contract_value_df[notional_copy_cols]
notional_df, contract_value_df = notional_copy.align(contract_value_df, join='left', axis='index')
# careful make sure orders of magnitude are same for the notional and the contract value
self._portfolio_signal_contracts = notional_df / contract_value_df
self._portfolio_signal_contracts.columns = self._portfolio_signal_notional.columns
self._portfolio_signal_trade_contracts = self._portfolio_signal_contracts - self._portfolio_signal_contracts.shift(1)
self._pnl = _pnl # individual signals P&L
# TODO FIX very slow - hence only calculate on demand
_pnl_trades = None
# _pnl_trades = calculations.calculate_individual_trade_gains(signal_df, _pnl)
self._pnl_trades = _pnl_trades
self._ret_stats_pnl = RetStats()
self._ret_stats_pnl.calculate_ret_stats(self._pnl, br.ann_factor)
self._portfolio.columns = ['Port']
self._ret_stats_portfolio = RetStats()
self._ret_stats_portfolio.calculate_ret_stats(self._portfolio, br.ann_factor)
self._cumpnl = calculations.create_mult_index(self._pnl) # individual signals cumulative P&L
self._cumpnl.columns = pnl_cols
self._cumportfolio = calculations.create_mult_index(self._portfolio) # portfolio cumulative P&L
self._cumportfolio.columns = ['Port']
def create_portfolio_weights(self, br, _pnl, method='mean'):
"""Calculates P&L of a trading strategy and statistics to be retrieved later
Parameters
----------
br : BacktestRequest
Parameters for the backtest specifying start date, finish data, transaction costs etc.
_pnl : pandas.DataFrame
Contains the daily P&L for the portfolio
method : String
'mean' - assumes equal weighting for each signal
'weighted' - can use predefined user weights (eg. if we assign weighting of 1, 1, 0.5, for three signals
the third signal will have a weighting of half versus the others)
Returns
-------
pandas.DataFrame
Contains the portfolio weights
"""
if method == 'mean':
weights_vector = numpy.ones(len(_pnl.columns))
elif method == 'weighted':
# get the weights for each asset
weights_vector = numpy.array([float(br.portfolio_combination[col]) for col in _pnl.columns])
# repeat this down for every day
weights_matrix = numpy.repeat(weights_vector[numpy.newaxis, :], len(_pnl.index), 0)
# where we don't have old price data, make the weights 0 there
ind = numpy.isnan(_pnl.values)
weights_matrix[ind] = 0
# the total weights will vary, as historically might not have all the assets trading
total_weights = numpy.sum(weights_matrix, axis=1)
# replicate across columns
total_weights = numpy.transpose(numpy.repeat(total_weights[numpy.newaxis, :], len(_pnl.columns), 0))
# to avoid divide by zero
total_weights[total_weights == 0.0] = 1.0
adjusted_weights_matrix = weights_matrix / total_weights
adjusted_weights_matrix[ind] = numpy.nan
return adjusted_weights_matrix
def get_backtest_output(self):
return
def get_pnl(self):
"""Gets P&L returns
Returns
-------
pandas.Dataframe
"""
return self._pnl
def get_pnl_trades(self):
"""Gets P&L of each individual trade per signal
Returns
-------
pandas.Dataframe
"""
if self._pnl_trades is None:
calculations = Calculations()
self._pnl_trades = calculations.calculate_individual_trade_gains(self._signal, self._pnl)
return self._pnl_trades
def get_pnl_desc(self):
"""Gets P&L return statistics in a string format
Returns
-------
str
"""
return self._ret_stats_signals.summary()
def get_pnl_ret_stats(self):
"""Gets P&L return statistics of individual strategies as class to be queried
Returns
-------
TimeSeriesDesc
"""
return self._ret_stats_pnl
def get_cumpnl(self):
"""Gets P&L as a cumulative time series of individual assets
Returns
-------
pandas.DataFrame
"""
return self._cumpnl
def get_cumportfolio(self):
"""Gets P&L as a cumulative time series of portfolio
Returns
-------
pandas.DataFrame
"""
return self._cumportfolio
def get_portfolio_pnl(self):
"""Gets portfolio returns in raw form (ie. not indexed into cumulative form)
Returns
-------
pandas.DataFrame
"""
return self._portfolio
def get_portfolio_pnl_desc(self):
"""Gets P&L return statistics of portfolio as string
Returns
-------
pandas.DataFrame
"""
return self._ret_stats_portfolio.summary()
def get_portfolio_pnl_ret_stats(self):
"""Gets P&L return statistics of portfolio as class to be queried
Returns
-------
RetStats
"""
return self._ret_stats_portfolio
def get_individual_leverage(self):
"""Gets leverage for each asset historically
Returns
-------
pandas.DataFrame
"""
return self._individual_leverage
def get_portfolio_leverage(self):
"""Gets the leverage for the portfolio
Returns
-------
pandas.DataFrame
"""
return self._portfolio_leverage
def get_portfolio_signal(self):
"""Gets the signals (with individual leverage & portfolio leverage) for each asset, which
equates to what we would trade in practice
Returns
-------
DataFrame
"""
return self._portfolio_signal
def get_portfolio_trade(self):
"""Gets the trades (with individual leverage & portfolio leverage) for each asset, which
we'd need to execute
Returns
-------
DataFrame
"""
return self._portfolio_trade
def get_portfolio_signal_notional(self):
"""Gets the signals (with individual leverage & portfolio leverage) for each asset, which
equates to what we would have a positions in practice, scaled by a notional amount we have already specified
Returns
-------
DataFrame
"""
return self._portfolio_signal_notional
def get_portfolio_trade_notional(self):
"""Gets the trades (with individual leverage & portfolio leverage) for each asset, which
we'd need to execute, scaled by a notional amount we have already specified
Returns
-------
DataFrame
"""
return self._portfolio_signal_trade_notional
def get_portfolio_signal_contracts(self):
"""Gets the signals (with individual leverage & portfolio leverage) for each asset, which
equates to what we would have a positions in practice, scaled by a notional amount and into contract sizes (eg. for futures)
which we need to specify in another dataframe
Returns
-------
DataFrame
"""
return self._portfolio_signal_contracts
def get_portfolio_trade_contracts(self):
"""Gets the trades (with individual leverage & portfolio leverage) for each asset, which
we'd need to execute, scaled by a notional amount we have already specified and into contract sizes (eg. for futures)
which we need to specify in another dataframe
Returns
-------
DataFrame
"""
return self._portfolio_signal_trade_contracts
def get_signal(self):
"""(with individual leverage, but excluding portfolio leverage) for each asset
Returns
-------
pandas.DataFrame
"""
return self._signal
class Backtest:
def __init__(self):
self.logger = LoggerManager().getLogger(__name__)
self._pnl = None
self._portfolio = None
return
def calculate_trading_PnL(self, br, asset_a_df, signal_df):
"""
calculate_trading_PnL - Calculates P&L of a trading strategy and statistics to be retrieved later
Parameters
----------
br : BacktestRequest
Parameters for the backtest specifying start date, finish data, transaction costs etc.
asset_a_df : pandas.DataFrame
Asset prices to be traded
signal_df : pandas.DataFrame
Signals for the trading strategy
"""
calculations = Calculations()
# signal_df.to_csv('e:/temp0.csv')
# make sure the dates of both traded asset and signal are aligned properly
asset_df, signal_df = asset_a_df.align(signal_df,
join='left',
axis='index')
# only allow signals to change on the days when we can trade assets
signal_df = signal_df.mask(numpy.isnan(
asset_df.values)) # fill asset holidays with NaN signals
signal_df = signal_df.fillna(method='ffill') # fill these down
asset_df = asset_df.fillna(method='ffill') # fill down asset holidays
returns_df = calculations.calculate_returns(asset_df)
tc = br.spot_tc_bp
signal_cols = signal_df.columns.values
returns_cols = returns_df.columns.values
pnl_cols = []
for i in range(0, len(returns_cols)):
pnl_cols.append(returns_cols[i] + " / " + signal_cols[i])
# do we have a vol target for individual signals?
if hasattr(br, 'signal_vol_adjust'):
if br.signal_vol_adjust is True:
risk_engine = RiskEngine()
if not (hasattr(br, 'signal_vol_resample_type')):
br.signal_vol_resample_type = 'mean'
if not (hasattr(br, 'signal_vol_resample_freq')):
br.signal_vol_resample_freq = None
leverage_df = risk_engine.calculate_leverage_factor(
returns_df, br.signal_vol_target,
br.signal_vol_max_leverage, br.signal_vol_periods,
br.signal_vol_obs_in_year, br.signal_vol_rebalance_freq,
br.signal_vol_resample_freq, br.signal_vol_resample_type)
signal_df = pandas.DataFrame(signal_df.values *
leverage_df.values,
index=signal_df.index,
columns=signal_df.columns)
self._individual_leverage = leverage_df # contains leverage of individual signal (before portfolio vol target)
_pnl = calculations.calculate_signal_returns_with_tc_matrix(signal_df,
returns_df,
tc=tc)
_pnl.columns = pnl_cols
# portfolio is average of the underlying signals: should we sum them or average them?
if hasattr(br, 'portfolio_combination'):
if br.portfolio_combination == 'sum':
portfolio = pandas.DataFrame(data=_pnl.sum(axis=1),
index=_pnl.index,
columns=['Portfolio'])
elif br.portfolio_combination == 'mean':
portfolio = pandas.DataFrame(data=_pnl.mean(axis=1),
index=_pnl.index,
columns=['Portfolio'])
else:
portfolio = pandas.DataFrame(data=_pnl.mean(axis=1),
index=_pnl.index,
columns=['Portfolio'])
portfolio_leverage_df = pandas.DataFrame(data=numpy.ones(
len(_pnl.index)),
index=_pnl.index,
columns=['Portfolio'])
# should we apply vol target on a portfolio level basis?
if hasattr(br, 'portfolio_vol_adjust'):
if br.portfolio_vol_adjust is True:
risk_engine = RiskEngine()
portfolio, portfolio_leverage_df = risk_engine.calculate_vol_adjusted_returns(
portfolio, br=br)
self._portfolio = portfolio
self._signal = signal_df # individual signals (before portfolio leverage)
self._portfolio_leverage = portfolio_leverage_df # leverage on portfolio
# multiply portfolio leverage * individual signals to get final position signals
length_cols = len(signal_df.columns)
leverage_matrix = numpy.repeat(
portfolio_leverage_df.values.flatten()[numpy.newaxis, :],
length_cols, 0)
# final portfolio signals (including signal & portfolio leverage)
self._portfolio_signal = pandas.DataFrame(data=numpy.multiply(
numpy.transpose(leverage_matrix), signal_df.values),
index=signal_df.index,
columns=signal_df.columns)
if hasattr(br, 'portfolio_combination'):
if br.portfolio_combination == 'sum':
pass
elif br.portfolio_combination == 'mean':
self._portfolio_signal = self._portfolio_signal / float(
length_cols)
else:
self._portfolio_signal = self._portfolio_signal / float(
length_cols)
self._pnl = _pnl # individual signals P&L
# TODO FIX very slow - hence only calculate on demand
_pnl_trades = None
# _pnl_trades = calculations.calculate_individual_trade_gains(signal_df, _pnl)
self._pnl_trades = _pnl_trades
self._ret_stats_pnl = RetStats()
self._ret_stats_pnl.calculate_ret_stats(self._pnl, br.ann_factor)
self._portfolio.columns = ['Port']
self._ret_stats_portfolio = RetStats()
self._ret_stats_portfolio.calculate_ret_stats(self._portfolio,
br.ann_factor)
self._cumpnl = calculations.create_mult_index(
self._pnl) # individual signals cumulative P&L
self._cumpnl.columns = pnl_cols
self._cumportfolio = calculations.create_mult_index(
self._portfolio) # portfolio cumulative P&L
self._cumportfolio.columns = ['Port']
def get_backtest_output(self):
return
def get_pnl(self):
"""
get_pnl - Gets P&L returns
Returns
-------
pandas.Dataframe
"""
return self._pnl
def get_pnl_trades(self):
"""
get_pnl_trades - Gets P&L of each individual trade per signal
Returns
-------
pandas.Dataframe
"""
if self._pnl_trades is None:
calculations = Calculations()
self._pnl_trades = calculations.calculate_individual_trade_gains(
self._signal, self._pnl)
return self._pnl_trades
def get_pnl_desc(self):
"""
get_pnl_desc - Gets P&L return statistics in a string format
Returns
-------
str
"""
return self._ret_stats_signals.summary()
def get_pnl_ret_stats(self):
"""
get_pnl_ret_stats - Gets P&L return statistics of individual strategies as class to be queried
Returns
-------
TimeSeriesDesc
"""
return self._ret_stats_pnl
def get_cumpnl(self):
"""
get_cumpnl - Gets P&L as a cumulative time series of individual assets
Returns
-------
pandas.DataFrame
"""
return self._cumpnl
def get_cumportfolio(self):
"""
get_cumportfolio - Gets P&L as a cumulative time series of portfolio
Returns
-------
pandas.DataFrame
"""
return self._cumportfolio
def get_portfolio_pnl(self):
"""
get_portfolio_pnl - Gets portfolio returns in raw form (ie. not indexed into cumulative form)
Returns
-------
pandas.DataFrame
"""
return self._portfolio
def get_portfolio_pnl_desc(self):
"""
get_portfolio_pnl_desc - Gets P&L return statistics of portfolio as string
Returns
-------
pandas.DataFrame
"""
return self._ret_stats_portfolio.summary()
def get_portfolio_pnl_ret_stats(self):
"""
get_portfolio_pnl_ret_stats - Gets P&L return statistics of portfolio as class to be queried
Returns
-------
RetStats
"""
return self._ret_stats_portfolio
def get_individual_leverage(self):
"""
get_individual_leverage - Gets leverage for each asset historically
Returns
-------
pandas.DataFrame
"""
return self._individual_leverage
def get_porfolio_leverage(self):
"""
get_portfolio_leverage - Gets the leverage for the portfolio
Returns
-------
pandas.DataFrame
"""
return self._portfolio_leverage
def get_porfolio_signal(self):
"""
get_portfolio_signal - Gets the signals (with individual leverage & portfolio leverage) for each asset, which
equates to what we would trade in practice
Returns
-------
DataFrame
"""
return self._portfolio_signal
def get_signal(self):
"""
get_signal - Gets the signals (with individual leverage, but excluding portfolio leverage) for each asset
Returns
-------
pandas.DataFrame
"""
return self._signal
class Backtest :
def __init__(self):
self.logger = LoggerManager().getLogger(__name__)
self._pnl = None
self._portfolio = None
return
def calculate_trading_PnL(self, br, asset_a_df, signal_df):
"""
calculate_trading_PnL - Calculates P&L of a trading strategy and statistics to be retrieved later
Parameters
----------
br : BacktestRequest
Parameters for the backtest specifying start date, finish data, transaction costs etc.
asset_a_df : pandas.DataFrame
Asset prices to be traded
signal_df : pandas.DataFrame
Signals for the trading strategy
"""
calculations = Calculations()
# make sure the dates of both traded asset and signal are aligned properly
asset_df, signal_df = asset_a_df.align(signal_df, join='left', axis = 'index')
# only allow signals to change on the days when we can trade assets
signal_df = signal_df.mask(numpy.isnan(asset_df.values)) # fill asset holidays with NaN signals
signal_df = signal_df.fillna(method='ffill') # fill these down
asset_df = asset_df.fillna(method='ffill') # fill down asset holidays
returns_df = calculations.calculate_returns(asset_df)
tc = br.spot_tc_bp
signal_cols = signal_df.columns.values
returns_cols = returns_df.columns.values
pnl_cols = []
for i in range(0, len(returns_cols)):
pnl_cols.append(returns_cols[i] + " / " + signal_cols[i])
# do we have a vol target for individual signals?
if hasattr(br, 'signal_vol_adjust'):
if br.signal_vol_adjust is True:
risk_engine = RiskEngine()
if not(hasattr(br, 'signal_vol_resample_type')):
br.signal_vol_resample_type = 'mean'
if not(hasattr(br, 'signal_vol_resample_freq')):
br.signal_vol_resample_freq = None
if not(hasattr(br, 'signal_vol_period_shift')):
br.signal_vol_period_shift = 0
leverage_df = risk_engine.calculate_leverage_factor(returns_df, br.signal_vol_target, br.signal_vol_max_leverage,
br.signal_vol_periods, br.signal_vol_obs_in_year,
br.signal_vol_rebalance_freq, br.signal_vol_resample_freq,
br.signal_vol_resample_type, period_shift=br.signal_vol_period_shift)
signal_df = pandas.DataFrame(
signal_df.values * leverage_df.values, index = signal_df.index, columns = signal_df.columns)
self._individual_leverage = leverage_df # contains leverage of individual signal (before portfolio vol target)
_pnl = calculations.calculate_signal_returns_with_tc_matrix(signal_df, returns_df, tc = tc)
_pnl.columns = pnl_cols
adjusted_weights_matrix = None
# portfolio is average of the underlying signals: should we sum them or average them?
if hasattr(br, 'portfolio_combination'):
if br.portfolio_combination == 'sum':
portfolio = pandas.DataFrame(data = _pnl.sum(axis = 1), index = _pnl.index, columns = ['Portfolio'])
elif br.portfolio_combination == 'mean':
portfolio = pandas.DataFrame(data = _pnl.mean(axis = 1), index = _pnl.index, columns = ['Portfolio'])
adjusted_weights_matrix = self.create_portfolio_weights(br, _pnl, method='mean')
elif isinstance(br.portfolio_combination, dict):
# get the weights for each asset
adjusted_weights_matrix = self.create_portfolio_weights(br, _pnl, method='weighted')
portfolio = pandas.DataFrame(data=(_pnl.values * adjusted_weights_matrix), index=_pnl.index)
is_all_na = pandas.isnull(portfolio).all(axis=1)
portfolio = pandas.DataFrame(portfolio.sum(axis = 1), columns = ['Portfolio'])
# overwrite days when every asset PnL was null is NaN with nan
portfolio[is_all_na] = numpy.nan
else:
portfolio = pandas.DataFrame(data = _pnl.mean(axis = 1), index = _pnl.index, columns = ['Portfolio'])
adjusted_weights_matrix = self.create_portfolio_weights(br, _pnl, method='mean')
portfolio_leverage_df = pandas.DataFrame(data = numpy.ones(len(_pnl.index)), index = _pnl.index, columns = ['Portfolio'])
# should we apply vol target on a portfolio level basis?
if hasattr(br, 'portfolio_vol_adjust'):
if br.portfolio_vol_adjust is True:
risk_engine = RiskEngine()
portfolio, portfolio_leverage_df = risk_engine.calculate_vol_adjusted_returns(portfolio, br = br)
self._portfolio = portfolio
self._signal = signal_df # individual signals (before portfolio leverage)
self._portfolio_leverage = portfolio_leverage_df # leverage on portfolio
# multiply portfolio leverage * individual signals to get final position signals
length_cols = len(signal_df.columns)
leverage_matrix = numpy.repeat(portfolio_leverage_df.values.flatten()[numpy.newaxis,:], length_cols, 0)
# final portfolio signals (including signal & portfolio leverage)
self._portfolio_signal = pandas.DataFrame(
data = numpy.multiply(numpy.transpose(leverage_matrix), signal_df.values),
index = signal_df.index, columns = signal_df.columns)
if hasattr(br, 'portfolio_combination'):
if br.portfolio_combination == 'sum':
pass
elif br.portfolio_combination == 'mean' or isinstance(br.portfolio_combination, dict):
self._portfolio_signal = pandas.DataFrame(data=(self._portfolio_signal.values * adjusted_weights_matrix),
index=self._portfolio_signal.index,
columns=self._portfolio_signal.columns)
else:
self._portfolio_signal = pandas.DataFrame(data=(self._portfolio_signal.values * adjusted_weights_matrix),
index=self._portfolio_signal.index,
columns=self._portfolio_signal.columns)
# calculate each period of trades
self._portfolio_trade = self._portfolio_signal - self._portfolio_signal.shift(1)
self._pnl = _pnl # individual signals P&L
# TODO FIX very slow - hence only calculate on demand
_pnl_trades = None
# _pnl_trades = calculations.calculate_individual_trade_gains(signal_df, _pnl)
self._pnl_trades = _pnl_trades
self._ret_stats_pnl = RetStats()
self._ret_stats_pnl.calculate_ret_stats(self._pnl, br.ann_factor)
self._portfolio.columns = ['Port']
self._ret_stats_portfolio = RetStats()
self._ret_stats_portfolio.calculate_ret_stats(self._portfolio, br.ann_factor)
self._cumpnl = calculations.create_mult_index(self._pnl) # individual signals cumulative P&L
self._cumpnl.columns = pnl_cols
self._cumportfolio = calculations.create_mult_index(self._portfolio) # portfolio cumulative P&L
self._cumportfolio.columns = ['Port']
def create_portfolio_weights(self, br, _pnl, method='mean'):
if method == 'mean':
weights_vector = numpy.ones(len(_pnl.columns))
elif method == 'weighted':
# get the weights for each asset
weights_vector = numpy.array([float(br.portfolio_combination[col]) for col in _pnl.columns])
# repeat this down for every day
weights_matrix = numpy.repeat(weights_vector[numpy.newaxis, :], len(_pnl.index), 0)
# where we don't have old price data, make the weights 0 there
ind = numpy.isnan(_pnl.values)
weights_matrix[ind] = 0
# the total weights will vary, as historically might not have all the assets trading
total_weights = numpy.sum(weights_matrix, axis=1)
# replicate across columns
total_weights = numpy.transpose(numpy.repeat(total_weights[numpy.newaxis, :], len(_pnl.columns), 0))
# to avoid divide by zero
total_weights[total_weights == 0.0] = 1.0
adjusted_weights_matrix = weights_matrix / total_weights
adjusted_weights_matrix[ind] = numpy.nan
return adjusted_weights_matrix
def get_backtest_output(self):
return
def get_pnl(self):
"""
get_pnl - Gets P&L returns
Returns
-------
pandas.Dataframe
"""
return self._pnl
def get_pnl_trades(self):
"""
get_pnl_trades - Gets P&L of each individual trade per signal
Returns
-------
pandas.Dataframe
"""
if self._pnl_trades is None:
calculations = Calculations()
self._pnl_trades = calculations.calculate_individual_trade_gains(self._signal, self._pnl)
return self._pnl_trades
def get_pnl_desc(self):
"""
get_pnl_desc - Gets P&L return statistics in a string format
Returns
-------
str
"""
return self._ret_stats_signals.summary()
def get_pnl_ret_stats(self):
"""
get_pnl_ret_stats - Gets P&L return statistics of individual strategies as class to be queried
Returns
-------
TimeSeriesDesc
"""
return self._ret_stats_pnl
def get_cumpnl(self):
"""
get_cumpnl - Gets P&L as a cumulative time series of individual assets
Returns
-------
pandas.DataFrame
"""
return self._cumpnl
def get_cumportfolio(self):
"""
get_cumportfolio - Gets P&L as a cumulative time series of portfolio
Returns
-------
pandas.DataFrame
"""
return self._cumportfolio
def get_portfolio_pnl(self):
"""
get_portfolio_pnl - Gets portfolio returns in raw form (ie. not indexed into cumulative form)
Returns
-------
pandas.DataFrame
"""
return self._portfolio
def get_portfolio_pnl_desc(self):
"""
get_portfolio_pnl_desc - Gets P&L return statistics of portfolio as string
Returns
-------
pandas.DataFrame
"""
return self._ret_stats_portfolio.summary()
def get_portfolio_pnl_ret_stats(self):
"""
get_portfolio_pnl_ret_stats - Gets P&L return statistics of portfolio as class to be queried
Returns
-------
RetStats
"""
return self._ret_stats_portfolio
def get_individual_leverage(self):
"""
get_individual_leverage - Gets leverage for each asset historically
Returns
-------
pandas.DataFrame
"""
return self._individual_leverage
def get_portfolio_leverage(self):
"""
get_portfolio_leverage - Gets the leverage for the portfolio
Returns
-------
pandas.DataFrame
"""
return self._portfolio_leverage
def get_portfolio_signal(self):
"""
get_portfolio_signal - Gets the signals (with individual leverage & portfolio leverage) for each asset, which
equates to what we would trade in practice
Returns
-------
DataFrame
"""
return self._portfolio_signal
def get_portfolio_trade(self):
"""
get_portfolio_trade - Gets the trades (with individual leverage & portfolio leverage) for each asset, which
we'd need to execute
Returns
-------
DataFrame
"""
return self._portfolio_trade
def get_signal(self):
"""
get_signal - Gets the signals (with individual leverage, but excluding portfolio leverage) for each asset
Returns
-------
pandas.DataFrame
"""
return self._signal