def calculate_diagnostic_trading_PnL(self, asset_a_df, signal_df, further_df = [], further_df_labels = []): """Calculates P&L table which can be used for debugging purposes, The table is populated with asset, signal and further dataframes provided by the user, can be used to check signalling methodology. It does not apply parameters such as transaction costs, vol adjusment and so on. Parameters ---------- asset_a_df : DataFrame Asset prices signal_df : DataFrame Trade signals (typically +1, -1, 0 etc) further_df : DataFrame Further dataframes user wishes to output in the diagnostic output (typically inputs for the signals) further_df_labels Labels to append to the further dataframes Returns ------- DataFrame with asset, trading signals and returns of the trading strategy for diagnostic purposes """ calculations = Calculations() asset_rets_df = calculations.calculate_returns(asset_a_df) strategy_rets = calculations.calculate_signal_returns(signal_df, asset_rets_df) reset_points = ((signal_df - signal_df.shift(1)).abs()) asset_a_df_entry = asset_a_df.copy(deep=True) asset_a_df_entry[reset_points == 0] = numpy.nan asset_a_df_entry = asset_a_df_entry.ffill() asset_a_df_entry.columns = [x + '_entry' for x in asset_a_df_entry.columns] asset_rets_df.columns = [x + '_asset_rets' for x in asset_rets_df.columns] strategy_rets.columns = [x + '_strat_rets' for x in strategy_rets.columns] signal_df.columns = [x + '_final_signal' for x in signal_df.columns] for i in range(0, len(further_df)): further_df[i].columns = [x + '_' + further_df_labels[i] for x in further_df[i].columns] flatten_df =[asset_a_df, asset_a_df_entry, asset_rets_df, strategy_rets, signal_df] for f in further_df: flatten_df.append(f) return calculations.pandas_outer_join(flatten_df)
def calculate_trading_PnL(self, br, asset_a_df, signal_df, contract_value_df = None): """Calculates P&L of a trading strategy and statistics to be retrieved later Calculates the P&L for each asset/signal combination and also for the finally strategy applying appropriate weighting in the portfolio, depending on predefined parameters, for example: static weighting for each asset static weighting for each asset + vol weighting for each asset static weighting for each asset + vol weighting for each asset + vol weighting for the portfolio Parameters ---------- br : BacktestRequest Parameters for the backtest specifying start date, finish data, transaction costs etc. asset_a_df : pandas.DataFrame Asset prices to be traded signal_df : pandas.DataFrame Signals for the trading strategy contract_value_df : pandas.DataFrame Daily size of contracts """ calculations = Calculations() # make sure the dates of both traded asset and signal are aligned properly asset_df, signal_df = asset_a_df.align(signal_df, join='left', axis = 'index') if (contract_value_df is not None): asset_df, contract_value_df = asset_df.align(contract_value_df, join='left', axis='index') contract_value_df = contract_value_df.fillna(method='ffill') # fill down asset holidays (we won't trade on these days) # non-trading days non_trading_days = numpy.isnan(asset_df.values) # only allow signals to change on the days when we can trade assets signal_df = signal_df.mask(non_trading_days) # fill asset holidays with NaN signals signal_df = signal_df.fillna(method='ffill') # fill these down tc = br.spot_tc_bp signal_cols = signal_df.columns.values asset_df_cols = asset_df.columns.values pnl_cols = [] for i in range(0, len(asset_df_cols)): pnl_cols.append(asset_df_cols[i] + " / " + signal_cols[i]) asset_df_copy = asset_df.copy() asset_df = asset_df.fillna(method='ffill') # fill down asset holidays (we won't trade on these days) returns_df = calculations.calculate_returns(asset_df) # apply a stop loss/take profit to every trade if this has been specified # do this before we start to do vol weighting etc. if hasattr(br, 'take_profit') and hasattr(br, 'stop_loss'): returns_df = calculations.calculate_returns(asset_df) temp_strategy_rets_df = calculations.calculate_signal_returns(signal_df, returns_df) trade_rets_df = calculations.calculate_cum_rets_trades(signal_df, temp_strategy_rets_df) # pre_signal_df = signal_df.copy() signal_df = calculations.calculate_risk_stop_signals(signal_df, trade_rets_df, br.stop_loss, br.take_profit) # make sure we can't trade where asset price is undefined and carry over signal signal_df = signal_df.mask(non_trading_days) # fill asset holidays with NaN signals signal_df = signal_df.fillna(method='ffill') # fill these down (when asset is not trading # signal_df.columns = [x + '_final_signal' for x in signal_df.columns] # for debugging purposes # if False: # signal_df_copy = signal_df.copy() # trade_rets_df_copy = trade_rets_df.copy() # # asset_df_copy.columns = [x + '_asset' for x in temp_strategy_rets_df.columns] # temp_strategy_rets_df.columns = [x + '_strategy_rets' for x in temp_strategy_rets_df.columns] # signal_df_copy.columns = [x + '_final_signal' for x in signal_df_copy.columns] # trade_rets_df_copy.columns = [x + '_cum_trade' for x in trade_rets_df_copy.columns] # # to_plot = calculations.pandas_outer_join([asset_df_copy, pre_signal_df, signal_df_copy, trade_rets_df_copy, temp_strategy_rets_df]) # to_plot.to_csv('test.csv') # do we have a vol target for individual signals? if hasattr(br, 'signal_vol_adjust'): if br.signal_vol_adjust is True: risk_engine = RiskEngine() if not(hasattr(br, 'signal_vol_resample_type')): br.signal_vol_resample_type = 'mean' if not(hasattr(br, 'signal_vol_resample_freq')): br.signal_vol_resample_freq = None if not(hasattr(br, 'signal_vol_period_shift')): br.signal_vol_period_shift = 0 leverage_df = risk_engine.calculate_leverage_factor(returns_df, br.signal_vol_target, br.signal_vol_max_leverage, br.signal_vol_periods, br.signal_vol_obs_in_year, br.signal_vol_rebalance_freq, br.signal_vol_resample_freq, br.signal_vol_resample_type, period_shift=br.signal_vol_period_shift) signal_df = pandas.DataFrame( signal_df.values * leverage_df.values, index = signal_df.index, columns = signal_df.columns) self._individual_leverage = leverage_df # contains leverage of individual signal (before portfolio vol target) _pnl = calculations.calculate_signal_returns_with_tc_matrix(signal_df, returns_df, tc = tc) _pnl.columns = pnl_cols adjusted_weights_matrix = None # portfolio is average of the underlying signals: should we sum them or average them? if hasattr(br, 'portfolio_combination'): if br.portfolio_combination == 'sum': portfolio = pandas.DataFrame(data = _pnl.sum(axis = 1), index = _pnl.index, columns = ['Portfolio']) elif br.portfolio_combination == 'mean': portfolio = pandas.DataFrame(data = _pnl.mean(axis = 1), index = _pnl.index, columns = ['Portfolio']) adjusted_weights_matrix = self.create_portfolio_weights(br, _pnl, method='mean') elif isinstance(br.portfolio_combination, dict): # get the weights for each asset adjusted_weights_matrix = self.create_portfolio_weights(br, _pnl, method='weighted') portfolio = pandas.DataFrame(data=(_pnl.values * adjusted_weights_matrix), index=_pnl.index) is_all_na = pandas.isnull(portfolio).all(axis=1) portfolio = pandas.DataFrame(portfolio.sum(axis = 1), columns = ['Portfolio']) # overwrite days when every asset PnL was null is NaN with nan portfolio[is_all_na] = numpy.nan else: portfolio = pandas.DataFrame(data = _pnl.mean(axis = 1), index = _pnl.index, columns = ['Portfolio']) adjusted_weights_matrix = self.create_portfolio_weights(br, _pnl, method='mean') portfolio_leverage_df = pandas.DataFrame(data = numpy.ones(len(_pnl.index)), index = _pnl.index, columns = ['Portfolio']) # should we apply vol target on a portfolio level basis? if hasattr(br, 'portfolio_vol_adjust'): if br.portfolio_vol_adjust is True: risk_engine = RiskEngine() portfolio, portfolio_leverage_df = risk_engine.calculate_vol_adjusted_returns(portfolio, br = br) self._portfolio = portfolio self._signal = signal_df # individual signals (before portfolio leverage) self._portfolio_leverage = portfolio_leverage_df # leverage on portfolio # multiply portfolio leverage * individual signals to get final position signals length_cols = len(signal_df.columns) leverage_matrix = numpy.repeat(portfolio_leverage_df.values.flatten()[numpy.newaxis,:], length_cols, 0) # final portfolio signals (including signal & portfolio leverage) self._portfolio_signal = pandas.DataFrame( data = numpy.multiply(numpy.transpose(leverage_matrix), signal_df.values), index = signal_df.index, columns = signal_df.columns) if hasattr(br, 'portfolio_combination'): if br.portfolio_combination == 'sum': pass elif br.portfolio_combination == 'mean' or isinstance(br.portfolio_combination, dict): self._portfolio_signal = pandas.DataFrame(data=(self._portfolio_signal.values * adjusted_weights_matrix), index=self._portfolio_signal.index, columns=self._portfolio_signal.columns) else: self._portfolio_signal = pandas.DataFrame(data=(self._portfolio_signal.values * adjusted_weights_matrix), index=self._portfolio_signal.index, columns=self._portfolio_signal.columns) # calculate each period of trades self._portfolio_trade = self._portfolio_signal - self._portfolio_signal.shift(1) self._portfolio_signal_notional = None self._portfolio_signal_trade_notional = None self._portfolio_signal_contracts = None self._portfolio_signal_trade_contracts = None # also create other measures of portfolio # portfolio & trades in terms of a predefined notional (in USD) # portfolio & trades in terms of contract sizes (particularly useful for futures) if hasattr(br, 'portfolio_notional_size'): # express positions in terms of the notional size specified self._portfolio_signal_notional = self._portfolio_signal * br.portfolio_notional_size self._portfolio_signal_trade_notional = self._portfolio_signal_notional - self._portfolio_signal_notional.shift(1) # get the positions in terms of the contract sizes notional_copy = self._portfolio_signal_notional.copy(deep=True) notional_copy_cols = [x.split('.')[0] for x in notional_copy.columns] notional_copy_cols = [x + '.contract-value' for x in notional_copy_cols] notional_copy.columns = notional_copy_cols contract_value_df = contract_value_df[notional_copy_cols] notional_df, contract_value_df = notional_copy.align(contract_value_df, join='left', axis='index') # careful make sure orders of magnitude are same for the notional and the contract value self._portfolio_signal_contracts = notional_df / contract_value_df self._portfolio_signal_contracts.columns = self._portfolio_signal_notional.columns self._portfolio_signal_trade_contracts = self._portfolio_signal_contracts - self._portfolio_signal_contracts.shift(1) self._pnl = _pnl # individual signals P&L # TODO FIX very slow - hence only calculate on demand _pnl_trades = None # _pnl_trades = calculations.calculate_individual_trade_gains(signal_df, _pnl) self._pnl_trades = _pnl_trades self._ret_stats_pnl = RetStats() self._ret_stats_pnl.calculate_ret_stats(self._pnl, br.ann_factor) self._portfolio.columns = ['Port'] self._ret_stats_portfolio = RetStats() self._ret_stats_portfolio.calculate_ret_stats(self._portfolio, br.ann_factor) self._cumpnl = calculations.create_mult_index(self._pnl) # individual signals cumulative P&L self._cumpnl.columns = pnl_cols self._cumportfolio = calculations.create_mult_index(self._portfolio) # portfolio cumulative P&L self._cumportfolio.columns = ['Port']