def forecast_turnover(self, instrument_code, rule_variation_name):
"""
Get the annualised turnover for a forecast/rule combination
:param instrument_code: instrument to get values for
:type instrument_code: str
:param rule_variation_name: rule to get values for
:type rule_variation_name: str
:returns: float
"""
use_pooled_turnover = str2Bool(
self.parent.config.forecast_cost_estimates['use_pooled_turnover'])
if use_pooled_turnover:
instrument_code_list = self.has_same_rules_as_code(instrument_code)
else:
instrument_code_list = [instrument_code]
turnover_for_SR = self.forecast_turnover_for_list(
instrument_code_list, rule_variation_name)
return turnover_for_SR
def forecast_scalar(xcross, window=250000, min_periods=500, backfill=True):
"""
Work out the scaling factor for xcross such that T*x has an abs value of 10
:param x:
:type x: pd.DataFrame TxN
:param span:
:type span: int
:param min_periods:
:returns: pd.DataFrame
"""
backfill = str2Bool(backfill) # in yaml will come in as text
# We don't allow this to be changed in config
target_abs_forecast = system_defaults['average_absolute_forecast']
# Take CS average first
# we do this before we get the final TS average otherwise get jumps in
# scalar
if xcross.shape[1] == 1:
x = xcross.abs().iloc[:, 0]
else:
x = xcross.ffill().abs().median(axis=1)
# now the TS
avg_abs_value = x.rolling(window=window, min_periods=min_periods).mean()
scaling_factor = target_abs_forecast / avg_abs_value
if backfill:
scaling_factor = scaling_factor.fillna(method="bfill")
return scaling_factor
def __init__(self,
data_as_df,
length_of_data=1,
ew_lookback=250,
boring_offdiag=0.99,
cleaning=True,
floor_at_zero=True,
**kwargs):
"""
Create an object to calculate correlations
We set up one of these with a set of data and parameters, and then call repeatedly
:param data_as_df: The dataframe of correlations
:param boring_offdiag: The off diagonal element to put into the matrix if data is absent
:param cleaning: Should we include fake values in the matrix so we don't need a warm up period?
:param floor_at_zero: Should we remove negative correlations?
:param ew_lookback: Lookback to use if exponential calculation used
:param length_of_data: Original length of data passed in (to correct for stacking of dataframe)
:return: np.array of correlation matrix
"""
self.cleaning = str2Bool(cleaning)
self.floor_at_zero = str2Bool(floor_at_zero)
## correct the lookback if we're jamming stuff together
self.ew_lookback_corrected = length_of_data * ew_lookback
size = data_as_df.shape[1]
self.corr_with_no_data = boring_corr_matrix(size, offdiag=np.nan)
self.corr_for_cleaning = boring_corr_matrix(size,
offdiag=boring_offdiag)
self.kwargs = kwargs
self.data_as_df = data_as_df
def get_costs(self, instrument_code):
"""
Get the relevant kinds of cost for an instrument
:param instrument_code: instrument to value for
:type instrument_code: str
:returns: 2 tuple
"""
use_SR_costs = str2Bool(self.parent.config.use_SR_costs)
if use_SR_costs:
return (self.get_SR_cost(instrument_code), None)
else:
return (None, self.get_cash_costs(instrument_code))
def get_forecast_correlation_matrices_from_code_list(self, codes_to_use):
"""
Returns a correlationList object which contains a history of correlation matricies
:param codes_to_use:
:type str:
:returns: correlation_list object
>>> from pysystemtrade.systems.tests.testdata import get_test_object_futures_with_rules_and_capping_estimate
>>> from pysystemtrade.systems.basesystem import System
>>> (accounts, fcs, rules, rawdata, data, config)=get_test_object_futures_with_rules_and_capping_estimate()
>>> system=System([rawdata, rules, fcs, accounts, ForecastCombineEstimated()], data, config)
>>> ans=system.combForecast.get_forecast_correlation_matrices("EDOLLAR")
>>> ans.corr_list[-1]
array([[ 1. , 0.1168699 , 0.08038547],
[ 0.1168699 , 1. , 0.86907623],
[ 0.08038547, 0.86907623, 1. ]])
>>> print(ans.columns)
['carry', 'ewmac16', 'ewmac8']
"""
# Get some useful stuff from the config
corr_params = copy(self.parent.config.forecast_correlation_estimate)
# do we pool our estimation?
pooling = str2Bool(corr_params.pop("pool_instruments"))
# which function to use for calculation
corr_func = resolve_function(corr_params.pop("func"))
self.log.terse("Calculating forecast correlations over %s" %
", ".join(codes_to_use))
forecast_data = [
self.get_all_forecasts(instr_code,
self.get_trading_rule_list(instr_code))
for instr_code in codes_to_use
]
# if we're not pooling passes a list of one
forecast_data = [forecast_ts.ffill() for forecast_ts in forecast_data]
return corr_func(forecast_data, **corr_params)
def correlation_calculator(data_for_estimate,
using_exponent=True,
min_periods=20,
ew_lookback=250):
"""
We generate a correlation from a pd.DataFrame, which could have been stacked up
:param data_for_estimate: Data to get correlations from
:type data_for_estimate: pd.DataFrame
:param using_exponent: Should we use exponential weighting? If not every item is weighted equally
:type using_exponent: bool
:param ew_lookback: Lookback, in periods, for exp. weighting
:type ew_lookback: int
:param min_periods: Minimum periods before we get a correlation
:type min_periods: int
:returns: 2-dim square np.array
"""
# These may come from config as str
using_exponent = str2Bool(using_exponent)
if using_exponent:
# If we have stacked there will be duplicate dates
# So we massage the span so it's correct
# This assumes the index is at least daily and on same timestamp
# This is an artifact of how we prepare the data
# Usual use for IDM, FDM calculation when whole data set is used
corrmat = data_for_estimate.ewm(
span=ew_lookback, min_periods=min_periods).corr(pairwise=True)
# only want the final one
corrmat = corrmat.values[-1]
else:
# Use normal correlation
# Usual use for bootstrapping when only have sub sample
corrmat = data_for_estimate.corr(min_periods=min_periods)
corrmat = corrmat.values
return corrmat
def get_forecast_correlation_matrices(self, instrument_code):
"""
Returns a correlationList object which contains a history of correlation matricies
:param instrument_code:
:type str:
:returns: correlation_list object
>>> from pysystemtrade.systems.tests.testdata import get_test_object_futures_with_rules_and_capping_estimate
>>> from pysystemtrade.systems.basesystem import System
>>> (accounts, fcs, rules, rawdata, data, config)=get_test_object_futures_with_rules_and_capping_estimate()
>>> system=System([rawdata, rules, fcs, accounts, ForecastCombineEstimated()], data, config)
>>> ans=system.combForecast.get_forecast_correlation_matrices("EDOLLAR")
>>> ans.corr_list[-1]
array([[ 1. , 0.1168699 , 0.08038547],
[ 0.1168699 , 1. , 0.86907623],
[ 0.08038547, 0.86907623, 1. ]])
>>> print(ans.columns)
['carry', 'ewmac16', 'ewmac8']
"""
# Get some useful stuff from the config
corr_params = copy(self.parent.config.forecast_correlation_estimate)
# do we pool our estimation?
pooling = str2Bool(corr_params.pop("pool_instruments"))
if pooling:
# find set of instruments with same trading rules as I have
codes_to_use = self.has_same_cheap_rules_as_code(instrument_code)
else:
codes_to_use = [instrument_code]
forecast_corr_list = self.get_forecast_correlation_matrices_from_code_list(
codes_to_use)
return forecast_corr_list
def use_estimated_div_mult(self):
return str2Bool(self.parent.config.use_forecast_div_mult_estimates)
def _use_estimated_weights(self):
return str2Bool(self.parent.config.use_forecast_weight_estimates)
def use_estimated_instrument_div_mult(self):
"""
It will determine if we use an estimate or a fixed class of object
"""
return str2Bool(self.parent.config.use_instrument_div_mult_estimates)
def __init__(
self,
data,
identifier=None,
parent=None,
frequency="W",
date_method="expanding",
rollyears=20,
method="bootstrap",
cleaning=True,
cost_multiplier=1.0,
apply_cost_weight=True,
ann_target_SR=TARGET_ANN_SR,
equalise_gross=False,
pool_gross_returns=False,
use_pooled_costs=False,
use_pooled_turnover=None, # not used
**passed_params):
"""
Set up optimiser
:param data: A dict of account curve dataframes, if not pooled data will only have one entry
:type data: dict
:param identifier: A dictionary key which refers to the element in data which we're optimising for
:type identifier: str
:param parent: The parent object, probably a system stage, which we get the log attribute from
:type parent: systemStage
:param frequency: Downsampling frequency. Must be "D", "W" or bigger
:type frequency: str
:param date_method: Method to pass to generate_fitting_dates
:type date_method: str
:param roll_years: If date_method is "rolling", number of years in window
:type roll_years: int
:param method: Method used for fitting, one of 'bootstrap', 'shrinkage', 'one_period'
:type method: str
:param cleaning: Do we clean weights so we don't need a warmup period?
:type cleaning: bool
:param equalise_gross: Should we equalise expected gross returns so that only costs affect weightings?
:type equalise_gross: bool
:param pool_gross_returns: Should we pool gross returns together?
:type pool_gross_returns: bool
:param use_pooled_costs: Should we pool costs together?
:type use_pooled_costs: bool
:param cost_multiplier: Multiply costs by this number
:type cost_multiplier: float
:param apply_cost_weight: Should we adjust our weightings to reflect costs?
:type apply_cost_weight: bool
:param *_estimate_params: dicts of **kwargs to pass to moments estimation, and optimisation functions
:returns: pd.DataFrame of weights
"""
if parent is None:
log = logtoscreen("optimiser")
else:
log = parent.log
setattr(self, "log", log)
# Because interaction of parameters is complex, display warnings
self.display_warnings(cost_multiplier, equalise_gross,
apply_cost_weight, method, **passed_params)
cleaning = str2Bool(cleaning)
optimise_params = copy(passed_params)
# annualisation
ANN_DICT = dict(D=BUSINESS_DAYS_IN_YEAR,
W=WEEKS_IN_YEAR,
M=MONTHS_IN_YEAR,
Y=1.0)
annualisation = ANN_DICT.get(frequency, 1.0)
self.set_up_data(data,
frequency=frequency,
equalise_gross=equalise_gross,
cost_multiplier=cost_multiplier,
annualisation=annualisation,
ann_target_SR=TARGET_ANN_SR,
use_pooled_costs=use_pooled_costs,
pool_gross_returns=pool_gross_returns,
identifier=identifier)
# A moments estimator works out the mean, vol, correlation
# Also stores annualisation factor and target SR (used for shrinkage
# and equalising)
moments_estimator = momentsEstimator(optimise_params, annualisation,
ann_target_SR)
# The optimiser instance will do the optimation once we have the
# appropriate data
optimiser = optimiserWithParams(method, optimise_params,
moments_estimator)
setattr(self, "optimiser", optimiser)
setattr(self, "frequency", frequency)
setattr(self, "method", method)
setattr(self, "equalise_gross", equalise_gross)
setattr(self, "cost_multiplier", cost_multiplier)
setattr(self, "annualisation", annualisation)
setattr(self, "date_method", date_method)
setattr(self, "rollyears", rollyears)
setattr(self, "cleaning", cleaning)
setattr(self, "apply_cost_weight", apply_cost_weight)