def __init__(self, optimise_params):
"""
Create an object which estimates the moments for a single period of data, according to the parameters
The parameters we need are popped from the config dict
:param optimise_params: Parameters for optimisation
:type optimise_params: dict
"""
corr_estimate_params=copy(optimise_params["correlation_estimate"])
mean_estimate_params=copy(optimise_params["mean_estimate"])
vol_estimate_params=copy(optimise_params["vol_estimate"])
corr_estimate_func=resolve_function(corr_estimate_params.pop("func"))
mean_estimate_func=resolve_function(mean_estimate_params.pop("func"))
vol_estimate_func=resolve_function(vol_estimate_params.pop("func"))
setattr(self, "corr_estimate_params", corr_estimate_params)
setattr(self, "mean_estimate_params", mean_estimate_params)
setattr(self, "vol_estimate_params", vol_estimate_params)
setattr(self, "corr_estimate_func", corr_estimate_func)
setattr(self, "mean_estimate_func", mean_estimate_func)
setattr(self, "vol_estimate_func", vol_estimate_func)
def __init__(self,
optimise_params,
annualisation=BUSINESS_DAYS_IN_YEAR,
ann_target_SR=.5):
"""
Create an object which estimates the moments for a single period of data, according to the parameters
The parameters we need are popped from the config dict
:param optimise_params: Parameters for optimisation
:type optimise_params: dict
"""
corr_estimate_params = copy(optimise_params["correlation_estimate"])
mean_estimate_params = copy(optimise_params["mean_estimate"])
vol_estimate_params = copy(optimise_params["vol_estimate"])
corr_estimate_func = resolve_function(corr_estimate_params.pop("func"))
mean_estimate_func = resolve_function(mean_estimate_params.pop("func"))
vol_estimate_func = resolve_function(vol_estimate_params.pop("func"))
setattr(self, "corr_estimate_params", corr_estimate_params)
setattr(self, "mean_estimate_params", mean_estimate_params)
setattr(self, "vol_estimate_params", vol_estimate_params)
setattr(self, "corr_estimate_func", corr_estimate_func)
setattr(self, "mean_estimate_func", mean_estimate_func)
setattr(self, "vol_estimate_func", vol_estimate_func)
period_target_SR = ann_target_SR / (annualisation**.5)
setattr(self, "annualisation", annualisation)
setattr(self, "period_target_SR", period_target_SR)
setattr(self, "ann_target_SR", ann_target_SR)
def __init__(self, method, optimise_params, moments_estimator):
"""
Create an object which does an optimisation for a single period, according to the parameters
:param optimise_params: Parameters for optimisation. Must contain "method"
:type optimise_params: dict
:param moments_estimator: An instance of a moments estimator
:type optimise_params: momentsEstimator
"""
fit_method_dict = dict(
one_period=markosolver,
bootstrap=bootstrap_portfolio,
shrinkage=opt_shrinkage,
equal_weights=equal_weights)
try:
opt_func = fit_method_dict[method]
except KeyError:
raise Exception("Fitting method %s unknown; try one of: %s " %
(method, ", ".join(fit_method_dict.keys())))
setattr(self, "opt_func", resolve_function(opt_func))
setattr(self, "params", optimise_params)
setattr(self, "moments_estimator", moments_estimator)
def calculation_of_raw_instrument_weights(self):
"""
Estimate the instrument weights
Done like this to expose calculations
:returns: TxK pd.DataFrame containing weights, columns are instrument names, T covers all
"""
def _calculation_of_raw_instrument_weights(system, NotUsed1, this_stage,
weighting_func, **weighting_params):
this_stage.log.terse("Calculating raw instrument weights")
instrument_codes = system.get_instrument_list()
weight_func = weighting_func(
log=this_stage.log.setup(
call="weighting"),
**weighting_params)
if weight_func.need_data():
if hasattr(system, "accounts"):
pandl = this_stage.pandl_across_subsystems()
(pandl_gross, pandl_costs) = decompose_group_pandl([pandl])
weight_func.set_up_data(
data_gross=pandl_gross, data_costs=pandl_costs)
else:
error_msg = "You need an accounts stage in the system to estimate instrument weights"
this_stage.log.critical(error_msg)
else:
# equal weights doesn't need data
positions = this_stage._get_all_subsystem_positions()
weight_func.set_up_data(weight_matrix=positions)
SR_cost_list = [this_stage.get_instrument_subsystem_SR_cost(
instr_code) for instr_code in instrument_codes]
weight_func.optimise(ann_SR_costs=SR_cost_list)
return weight_func
# Get some useful stuff from the config
weighting_params = copy(self.parent.config.instrument_weight_estimate)
# which function to use for calculation
weighting_func = resolve_function(weighting_params.pop("func"))
calcs_of_instrument_weights = self.parent.calc_or_cache(
'calculation_of_raw_instrument_weights', ALL_KEYNAME,
_calculation_of_raw_instrument_weights,
self, weighting_func, **weighting_params)
return calcs_of_instrument_weights
def daily_returns_volatility(self, instrument_code):
"""
Gets volatility of daily returns (not % returns)
This is done using a user defined function
We get this from:
the configuration object
or if not found, system.defaults.py
The dict must contain func key; anything else is optional
:param instrument_code: Instrument to get prices for
:type trading_rules: str
:returns: Tx1 pd.DataFrame
>>> from systems.tests.testdata import get_test_object
>>> from systems.basesystem import System
>>>
>>> (rawdata, data, config)=get_test_object()
>>> system=System([rawdata], data)
>>> ## uses defaults
>>> system.rawdata.daily_returns_volatility("EDOLLAR").tail(2)
vol
2015-12-10 0.054145
2015-12-11 0.058522
>>>
>>> from sysdata.configdata import Config
>>> config=Config("systems.provided.example.exampleconfig.yaml")
>>> system=System([rawdata], data, config)
>>> system.rawdata.daily_returns_volatility("EDOLLAR").tail(2)
vol
2015-12-10 0.054145
2015-12-11 0.058522
>>>
>>> config=Config(dict(volatility_calculation=dict(func="syscore.algos.robust_vol_calc", days=200)))
>>> system2=System([rawdata], data, config)
>>> system2.rawdata.daily_returns_volatility("EDOLLAR").tail(2)
vol
2015-12-10 0.057946
2015-12-11 0.058626
"""
self.log.msg(
"Calculating daily volatility for %s" % instrument_code,
instrument_code=instrument_code)
system = self.parent
dailyreturns = self.daily_returns(instrument_code)
volconfig = copy(system.config.volatility_calculation)
# volconfig contains 'func' and some other arguments
# we turn func which could be a string into a function, and then
# call it with the other ags
volfunction = resolve_function(volconfig.pop('func'))
vol = volfunction(dailyreturns, **volconfig)
return vol
def _get_forecast_scalar_estimated_from_instrument_list(
self, instrument_code, rule_variation_name,
forecast_scalar_config):
"""
Get the scalar to apply to raw forecasts
If not cached, these are estimated from past forecasts
:param instrument_code: instrument code, or ALL_KEYNAME if pooling
:type str:
:param rule_variation_name:
:type str: name of the trading rule variation
:param forecast_scalar_config:
:type dict: relevant part of the config
:returns: float
"""
# The config contains 'func' and some other arguments
# we turn func which could be a string into a function, and then
# call it with the other ags
scalar_function = resolve_function(forecast_scalar_config.pop('func'))
"""
instrument_list contains multiple things, might pool everything across
all instruments
"""
if instrument_code == ALL_KEYNAME:
# pooled, same for all instruments
instrument_list = self.parent.get_instrument_list()
else:
## not pooled
instrument_list = [instrument_code]
self.log.msg(
"Getting forecast scalar for %s over %s" %
(rule_variation_name, ", ".join(instrument_list)),
rule_variation_name=rule_variation_name)
# Get forecasts for each instrument
forecast_list = [
self.get_raw_forecast(instrument_code, rule_variation_name)
for instrument_code in instrument_list
]
cs_forecasts = pd.concat(forecast_list, axis=1)
# an example of a scaling function is syscore.algos.forecast_scalar
# must return thing the same size as cs_forecasts
scaling_factor = scalar_function(cs_forecasts,
**forecast_scalar_config)
return scaling_factor
def __init__(self, optimise_params, annualisation=BUSINESS_DAYS_IN_YEAR,
ann_target_SR=.5):
corr_estimate_params=copy(optimise_params["correlation_estimate"])
mean_estimate_params=copy(optimise_params["mean_estimate"])
vol_estimate_params=copy(optimise_params["vol_estimate"])
corr_estimate_func=resolve_function(corr_estimate_params.pop("func"))
mean_estimate_func=resolve_function(mean_estimate_params.pop("func"))
vol_estimate_func=resolve_function(vol_estimate_params.pop("func"))
setattr(self, "corr_estimate_params", corr_estimate_params)
setattr(self, "mean_estimate_params", mean_estimate_params)
setattr(self, "vol_estimate_params", vol_estimate_params)
setattr(self, "corr_estimate_func", corr_estimate_func)
setattr(self, "mean_estimate_func", mean_estimate_func)
setattr(self, "vol_estimate_func", vol_estimate_func)
period_target_SR = ann_target_SR / (annualisation**.5)
setattr(self, "annualisation", annualisation)
setattr(self, "period_target_SR", period_target_SR)
setattr(self, "ann_target_SR", ann_target_SR)
def get_instrument_correlation_matrix(self):
"""
Returns a correlationList object which contains a history of correlation matricies
:returns: correlation_list object
>>> from systems.tests.testdata import get_test_object_futures_with_pos_sizing_estimates
>>> from systems.basesystem import System
>>> (account, posobject, combobject, capobject, rules, rawdata, data, config)=get_test_object_futures_with_pos_sizing_estimates()
>>> system=System([rawdata, rules, posobject, combobject, capobject,PortfoliosEstimated(), account], data, config)
>>> system.config.forecast_weight_estimate["method"]="shrinkage" ## speed things up
>>> system.config.forecast_weight_estimate["date_method"]="in_sample" ## speed things up
>>> system.config.instrument_weight_estimate["date_method"]="in_sample" ## speed things up
>>> system.config.instrument_weight_estimate["method"]="shrinkage" ## speed things up
>>> ans=system.portfolio.get_instrument_correlation_matrix()
>>> ans.corr_list[-1]
array([[ 1. , 0.56981346, 0.62458477],
[ 0.56981346, 1. , 0.88087893],
[ 0.62458477, 0.88087893, 1. ]])
>>> print(ans.corr_list[0])
[[ 1. 0.99 0.99]
[ 0.99 1. 0.99]
[ 0.99 0.99 1. ]]
>>> print(ans.corr_list[10])
[[ 1. 0.99 0.99 ]
[ 0.99 1. 0.78858156]
[ 0.99 0.78858156 1. ]]
"""
self.log.terse("Calculating instrument correlations")
system = self.parent
# Get some useful stuff from the config
corr_params = copy(system.config.instrument_correlation_estimate)
# which function to use for calculation
corr_func = resolve_function(corr_params.pop("func"))
if hasattr(system, "accounts"):
pandl = self.pandl_across_subsystems().to_frame()
else:
error_msg = "You need an accounts stage in the system to estimate instrument correlations"
self.log.critical(error_msg)
# Need to resample here, because the correlation function won't do
# it properly (doesn't know it's dealing with returns data)
frequency = corr_params['frequency']
pandl = pandl.cumsum().resample(frequency).last().diff()
# The subsequent resample inside the correlation function will have no effect
return corr_func(pandl, **corr_params)
def _daily_returns_volatility(system, instrument_code, this_stage):
this_stage.log.msg("Calculating daily volatility for %s" % instrument_code, instrument_code=instrument_code)
dailyreturns = this_stage.daily_returns(instrument_code)
volconfig=copy(system.config.volatility_calculation)
# volconfig contains 'func' and some other arguments
# we turn func which could be a string into a function, and then
# call it with the other ags
volfunction = resolve_function(volconfig.pop('func'))
vol = volfunction(dailyreturns, **volconfig)
return vol
def _daily_returns_volatility(system, instrument_code, this_stage):
print(__file__ + ":" + str(inspect.getframeinfo(inspect.currentframe())[:3][1]) + ":" +"Calculating daily volatility for %s" % instrument_code)
dailyreturns = this_stage.daily_returns(instrument_code)
volconfig=copy(system.config.volatility_calculation)
# volconfig contains 'func' and some other arguments
# we turn func which could be a string into a function, and then
# call it with the other ags
volfunction = resolve_function(volconfig.pop('func'))
vol = volfunction(dailyreturns, **volconfig)
return vol
def _get_forecast_div_multiplier(system, instrument_code, this_stage):
print(__file__ + ":" + str(inspect.getframeinfo(inspect.currentframe())[:3][1]) + ":" +"Calculating forecast div multiplier for %s" % instrument_code)
## Get some useful stuff from the config
div_mult_params=copy(system.config.forecast_div_mult_estimate)
idm_func=resolve_function(div_mult_params.pop("func"))
correlation_list_object=this_stage.get_forecast_correlation_matrices(instrument_code)
weight_df=this_stage.get_forecast_weights(instrument_code)
ts_fdm=idm_func(correlation_list_object, weight_df, **div_mult_params)
return ts_fdm
def _get_instrument_div_multiplier(system, NotUsed, this_stage):
this_stage.log.terse("Calculating instrument div. multiplier")
## Get some useful stuff from the config
div_mult_params=copy(system.config.instrument_div_mult_estimate)
idm_func=resolve_function(div_mult_params.pop("func"))
correlation_list_object=this_stage.get_instrument_correlation_matrix()
weight_df=this_stage.get_instrument_weights()
ts_idm=idm_func(correlation_list_object, weight_df, **div_mult_params)
return ts_idm
def _get_forecast_div_multiplier(system, instrument_code, this_stage):
this_stage.log.terse("Calculating forecast div multiplier for %s" % instrument_code,
instrument_code=instrument_code)
## Get some useful stuff from the config
div_mult_params=copy(system.config.forecast_div_mult_estimate)
idm_func=resolve_function(div_mult_params.pop("func"))
correlation_list_object=this_stage.get_forecast_correlation_matrices(instrument_code)
weight_df=this_stage.get_forecast_weights(instrument_code)
ts_fdm=idm_func(correlation_list_object, weight_df, **div_mult_params)
return ts_fdm
def calculation_of_raw_instrument_weights(self):
"""
Estimate the instrument weights
Done like this to expose calculations
:returns: TxK pd.DataFrame containing weights, columns are instrument names, T covers all
"""
def _calculation_of_raw_instrument_weights(system, NotUsed1, this_stage,
weighting_func, **weighting_params):
this_stage.log.terse("Calculating raw instrument weights")
instrument_codes=system.get_instrument_list()
if hasattr(system, "accounts"):
pandl_subsystems=[this_stage.pandl_for_subsystem(code)
for code in instrument_codes]
else:
error_msg="You need an accounts stage in the system to estimate instrument weights"
this_stage.log.critical(error_msg)
pandl=pd.concat(pandl_subsystems, axis=1)
pandl.columns=instrument_codes
instrument_weight_results=weighting_func(pandl, log=self.log.setup(call="weighting"), **weighting_params)
return instrument_weight_results
## Get some useful stuff from the config
weighting_params=copy(self.parent.config.instrument_weight_estimate)
## which function to use for calculation
weighting_func=resolve_function(weighting_params.pop("func"))
calcs_of_instrument_weights = self.parent.calc_or_cache(
'calculation_of_raw_instrument_weights', ALL_KEYNAME,
_calculation_of_raw_instrument_weights,
self, weighting_func, **weighting_params)
return calcs_of_instrument_weights
def capital_multiplier(self, delayfill=True, roundpositions=False):
"""
Get a capital multiplier
:param delayfill: Lag fills by one day
:type delayfill: bool
:param roundpositions: Round positions to whole contracts
:type roundpositions: bool
:returns: pd.Series
"""
system = self.parent
capmult_params = copy(system.config.capital_multiplier)
capmult_func = resolve_function(capmult_params.pop("func"))
capmult = capmult_func(system, **capmult_params)
capmult = capmult.reindex(self.portfolio().index).ffill()
return capmult
def _daily_returns_volatility(system, instrument_code, this_stage):
dailyreturns = this_stage.daily_returns(instrument_code)
try:
volconfig = copy(system.config.volatility_calculation)
identify_error = "inherited from config object"
except:
volconfig = copy(system_defaults['volatility_calculation'])
identify_error = "found in system.defaults.py"
if "func" not in volconfig:
raise Exception(
"The volconfig dict (%s) needs to have a 'func' key" % identify_error)
# volconfig contains 'func' and some other arguments
# we turn func which could be a string into a function, and then
# call it with the other ags
volfunction = resolve_function(volconfig.pop('func'))
vol = volfunction(dailyreturns, **volconfig)
return vol
def get_estimated_instrument_diversification_multiplier(self):
"""
Estimate the diversification multiplier for the portfolio
Estimated from correlations and weights
:returns: Tx1 pd.DataFrame
>>> from systems.tests.testdata import get_test_object_futures_with_pos_sizing_estimates
>>> from systems.basesystem import System
>>> (account, posobject, combobject, capobject, rules, rawdata, data, config)=get_test_object_futures_with_pos_sizing_estimates()
>>> system=System([rawdata, rules, posobject, combobject, capobject,PortfoliosEstimated(), account], data, config)
>>> system.config.forecast_weight_estimate["method"]="shrinkage" ## speed things up
>>> system.config.forecast_weight_estimate["date_method"]="in_sample" ## speed things up
>>> system.config.instrument_weight_estimate["date_method"]="in_sample" ## speed things up
>>> system.config.instrument_weight_estimate["method"]="shrinkage" ## speed things up
>>> system.portfolio.get_instrument_diversification_multiplier().tail(3)
IDM
2015-12-09 1.133220
2015-12-10 1.133186
2015-12-11 1.133153
"""
self.log.terse("Calculating instrument div. multiplier")
# Get some useful stuff from the config
div_mult_params = copy(self.parent.config.instrument_div_mult_estimate)
idm_func = resolve_function(div_mult_params.pop("func"))
correlation_list_object = self.get_instrument_correlation_matrix()
weight_df = self.get_instrument_weights()
ts_idm = idm_func(correlation_list_object, weight_df,
**div_mult_params)
return ts_idm
def calculation_of_raw_instrument_weights(self):
"""
Estimate the instrument weights
Done like this to expose calculations
:returns: TxK pd.DataFrame containing weights, columns are instrument names, T covers all
"""
# Get some useful stuff from the config
weighting_params = copy(self.parent.config.instrument_weight_estimate)
# which function to use for calculation
weighting_func = resolve_function(weighting_params.pop("func"))
system = self.parent
self.log.terse("Calculating raw instrument weights")
instrument_codes = system.get_instrument_list()
if hasattr(system, "accounts"):
pandl = self.pandl_across_subsystems()
else:
error_msg = "You need an accounts stage in the system to estimate instrument weights"
self.log.critical(error_msg)
# The optimiser is set up for pooling, but we're not going to do that
# Create a single fake set of return data
data = dict(instrument_pandl = pandl)
weight_func = weighting_func(data, identifier ="instrument_pandl", parent=self,
**weighting_params)
weight_func.optimise()
return weight_func
def get_instrument_correlation_matrix(self):
"""
Returns a correlationList object which contains a history of correlation matricies
:returns: correlation_list object
>>> from systems.tests.testdata import get_test_object_futures_with_pos_sizing_estimates
>>> from systems.basesystem import System
>>> (account, posobject, combobject, capobject, rules, rawdata, data, config)=get_test_object_futures_with_pos_sizing_estimates()
>>> system=System([rawdata, rules, posobject, combobject, capobject,PortfoliosEstimated(), account], data, config)
>>> system.config.forecast_weight_estimate["method"]="shrinkage" ## speed things up
>>> system.config.forecast_weight_estimate["date_method"]="in_sample" ## speed things up
>>> system.config.instrument_weight_estimate["date_method"]="in_sample" ## speed things up
>>> system.config.instrument_weight_estimate["method"]="shrinkage" ## speed things up
>>> ans=system.portfolio.get_instrument_correlation_matrix()
>>> ans.corr_list[-1]
array([[ 1. , 0.50156603, 0.56866684],
[ 0.50156603, 1. , 0.88358678],
[ 0.56866684, 0.88358678, 1. ]])
>>> print(ans.corr_list[0])
[[ 1. 0.99 0.99]
[ 0.99 1. 0.99]
[ 0.99 0.99 1. ]]
>>> print(ans.corr_list[10])
[[ 1. 0.99 0.99 ]
[ 0.99 1. 0.76599709]
[ 0.99 0.76599709 1. ]]
"""
def _get_instrument_correlation_matrix(system, NotUsed, this_stage,
corr_func, **corr_params):
this_stage.log.terse("Calculating instrument correlations")
instrument_codes=system.get_instrument_list()
if hasattr(system, "accounts"):
pandl=this_stage.pandl_across_subsystems()
else:
error_msg="You need an accounts stage in the system to estimate instrument correlations"
this_stage.log.critical(error_msg)
## Need to resample here, because the correlation function won't do it properly
frequency=corr_params['frequency']
pandl=pandl.cumsum().resample(frequency).diff()
return corr_func(pandl, log=this_stage.log.setup(call="correlation"), **corr_params)
## Get some useful stuff from the config
corr_params=copy(self.parent.config.instrument_correlation_estimate)
## which function to use for calculation
corr_func=resolve_function(corr_params.pop("func"))
## _get_instrument_correlation_matrix: function to call if we don't find in cache
## self: this_system stage object
## func: function to call to calculate correlations
## **corr_params: parameters to pass to correlation function
##
forecast_corr_list = self.parent.calc_or_cache(
'get_instrument_correlation_matrix', ALL_KEYNAME,
_get_instrument_correlation_matrix,
self, corr_func, **corr_params)
return forecast_corr_list
def calculation_of_raw_forecast_weights(self, instrument_code):
"""
Estimate the forecast weights for this instrument
We store this intermediate step to expose the calculation object
:param instrument_code:
:type str:
:returns: TxK pd.DataFrame containing weights, columns are trading rule variation names, T covers all
"""
def _calculation_of_raw_forecast_weights(system, NotUsed1, NotUsed2, this_stage,
codes_to_use, weighting_func, **weighting_params):
this_stage.log.terse("Calculating raw forecast weights over %s" % ", ".join(codes_to_use))
if hasattr(system, "accounts"):
pandl_forecasts=[this_stage.pandl_for_instrument_rules_unweighted(code)
for code in codes_to_use]
else:
error_msg="You need an accounts stage in the system to estimate forecast weights"
this_stage.log.critical(error_msg)
output=weighting_func(pandl_forecasts, log=self.log.setup(call="weighting"), **weighting_params)
return output
## Get some useful stuff from the config
weighting_params=copy(self.parent.config.forecast_weight_estimate)
## do we pool our estimation?
pooling=str2Bool(weighting_params.pop("pool_instruments"))
## which function to use for calculation
weighting_func=resolve_function(weighting_params.pop("func"))
if pooling:
## find set of instruments with same trading rules as I have
codes_to_use=self._has_same_rules_as_code(instrument_code)
instrument_code_ref=ALL_KEYNAME
## We
label='_'.join(codes_to_use)
else:
codes_to_use=[instrument_code]
label=instrument_code
instrument_code_ref=instrument_code
##
## label: how we identify this thing in the cache
## instrument_code_ref: eithier the instrument code, or 'all markets' if pooling
## _get_raw_forecast_weights: function to call if we don't find in cache
## self: this_system stage object
## codes_to_use: instrument codes to get data for
## weighting_func: function to call to calculate weights
## **weighting_params: parameters to pass to weighting function
##
raw_forecast_weights_calcs = self.parent.calc_or_cache_nested(
'calculation_of_raw_forecast_weights', instrument_code_ref, label,
_calculation_of_raw_forecast_weights,
self, codes_to_use, weighting_func, **weighting_params)
return raw_forecast_weights_calcs
def __init__(self, rule, data=list(), other_args=dict()):
"""
Create a trading rule from a function
Functions must be of the form function(*dataargs, **kwargs), where
*dataargs are unnamed data items, and **kwargs are named configuration
items data, an ordered list of strings identifying data to be used
(default, just price) other_args: a dictionary of named arguments to be
passed to the trading rule
:param rule: Trading rule to be created
:type trading_rules:
The following describe a rule completely (ignore data and other_args arguments)
3-tuple ; containing (function, data, other_args)
dict (containing key "function", and optionally keys "other_args" and "data")
TradingRule (object is created out of this rule)
The following will be combined with the data and other_args arguments
to produce a complete TradingRule:
Other callable function
str (with path to function eg
"systems.provide.example.rules.ewmac_forecast_with_defaults")
:param data: (list of) str pointing to location of inputs in a system method call (eg "data.get_instrument_price")
(Either passed in separately, or as part of a TradingRule, 3-tuple, or dict object)
:type data: single str, or list of str
:param other_args: Other named arguments to be passed to trading rule function
(Either passed in separately , or as part of a TradingRule, 3-tuple, or dict object)
:type other_args: dict
:returns: single Tradingrule object
"""
if hasallattr(rule, ["function", "data", "other_args"]):
# looks like it is already a trading rule
(rule_function, data, other_args) = (rule.function, rule.data,
rule.other_args)
elif isinstance(rule, tuple):
if len(data) > 0 or len(other_args) > 0:
print(
"WARNING: Creating trade rule with 'rule' tuple argument, ignoring data and/or other args"
)
if len(rule) != 3:
raise Exception(
"Creating trading rule with a tuple, must be length 3 exactly (function/name, data [...], args dict(...))"
)
(rule_function, data, other_args) = rule
elif isinstance(rule, dict):
if len(data) > 0 or len(other_args) > 0:
print(
"WARNING: Creating trade rule with 'rule' dict argument, ignoring data and/or other args"
)
try:
rule_function = rule['function']
except KeyError:
raise Exception(
"If you specify a TradingRule as a dict it has to contain a 'function' keyname"
)
if "data" in rule:
data = rule['data']
else:
data = []
if "other_args" in rule:
other_args = rule['other_args']
else:
other_args = dict()
else:
rule_function = rule
# turn string into a callable function if required
rule_function = resolve_function(rule_function)
if isinstance(data, str):
# turn into a 1 item list or wont' get parsed properly
data = [data]
setattr(self, "function", rule_function)
setattr(self, "data", data)
setattr(self, "other_args", other_args)
def calculation_of_raw_forecast_weights_for_instrument(
self, instrument_code):
"""
Does an optimisation for a single instrument
We do this if we can't do the special case of a pooled optimisation
Estimate the forecast weights for this instrument
We store this intermediate step to expose the calculation object
:param instrument_code:
:type str:
:returns: TxK pd.DataFrame containing weights, columns are trading rule variation names, T covers all
"""
def _calculation_of_raw_forecast_weights(system, instrument_code,
this_stage, codes_to_use,
weighting_func, pool_costs,
**weighting_params):
this_stage.log.terse(
"Calculating raw forecast weights for %s, over %s" %
(instrument_code, ", ".join(codes_to_use)))
rule_list = self.apply_cost_weighting(instrument_code)
weight_func = weighting_func(log=self.log.setup(call="weighting"),
**weighting_params)
if weight_func.need_data():
## returns a list of accountCurveGroups
pandl_forecasts = [
this_stage.get_returns_for_optimisation(code)
for code in codes_to_use
]
## the current curve is special
pandl_forecasts_this_code = this_stage.get_returns_for_optimisation(
instrument_code)
## have to decode these
## returns two lists of pd.DataFrames
(pandl_forecasts_gross,
pandl_forecasts_costs) = decompose_group_pandl(
pandl_forecasts,
pandl_forecasts_this_code,
pool_costs=pool_costs)
## The weighting function requires two lists of pd.DataFrames, one gross, one for costs
weight_func.set_up_data(data_gross=pandl_forecasts_gross,
data_costs=pandl_forecasts_costs)
else:
## in the case of equal weights, don't need data
forecasts = this_stage.get_all_forecasts(
instrument_code, rule_list)
weight_func.set_up_data(weight_matrix=forecasts)
SR_cost_list = [
this_stage.get_SR_cost_for_instrument_forecast(
instrument_code, rule_variation_name)
for rule_variation_name in rule_list
]
weight_func.optimise(ann_SR_costs=SR_cost_list)
return weight_func
## Get some useful stuff from the config
weighting_params = copy(self.parent.config.forecast_weight_estimate)
## do we pool our estimation?
pooling_returns = str2Bool(
self.parent.config.forecast_weight_estimate["pool_gross_returns"])
pool_costs = str2Bool(
self.parent.config.forecast_cost_estimates["use_pooled_costs"])
## which function to use for calculation
weighting_func = resolve_function(weighting_params.pop("func"))
if pooling_returns:
## find set of instruments with same trading rules as I have
codes_to_use = self.has_same_cheap_rules_as_code(instrument_code)
else:
codes_to_use = [instrument_code]
##
## _get_raw_forecast_weights: function to call if we don't find in cache
## self: this_system stage object
## codes_to_use: instrument codes to get data for
## weighting_func: function to call to calculate weights
## **weighting_params: parameters to pass to weighting function
##
raw_forecast_weights_calcs = self.parent.calc_or_cache(
'calculation_of_raw_forecast_weights', instrument_code,
_calculation_of_raw_forecast_weights, self, codes_to_use,
weighting_func, pool_costs, **weighting_params)
return raw_forecast_weights_calcs
pass
def get_forecast_scalar(self, instrument_code, rule_variation_name):
"""
Get the scalar to apply to raw forecasts
If not cached, these are estimated from past forecasts
If configuration variable pool_forecasts_for_scalar is "True", then we do this across instruments.
:param instrument_code:
:type str:
:param rule_variation_name:
:type str: name of the trading rule variation
:returns: float
>>> from systems.tests.testdata import get_test_object_futures_with_rules
>>> from systems.basesystem import System
>>> (rules, rawdata, data, config)=get_test_object_futures_with_rules()
>>> system1=System([rawdata, rules, ForecastScaleCapEstimated()], data, config)
>>>
>>> ## From default
>>> system1.forecastScaleCap.get_forecast_scalar("EDOLLAR", "ewmac8").tail(3)
scale_factor
2015-12-09 5.849888
2015-12-10 5.850474
2015-12-11 5.851091
>>> system1.forecastScaleCap.get_capped_forecast("EDOLLAR", "ewmac8").tail(3)
ewmac8
2015-12-09 0.645585
2015-12-10 -0.210377
2015-12-11 0.961821
>>>
>>> ## From config
>>> scale_config=dict(pool_instruments=False)
>>> config.forecast_scalar_estimate=scale_config
>>> system3=System([rawdata, rules, ForecastScaleCapEstimated()], data, config)
>>> system3.forecastScaleCap.get_forecast_scalar("EDOLLAR", "ewmac8").tail(3)
scale_factor
2015-12-09 5.652174
2015-12-10 5.652833
2015-12-11 5.653444
>>>
"""
def _get_forecast_scalar(
system, Not_Used, rule_variation_name, this_stage, instrument_list,
scalar_function, forecast_scalar_config):
"""
instrument_list contains multiple things, pools everything across all instruments
"""
print(__file__ + ":" + str(inspect.getframeinfo(inspect.currentframe())[:3][1]) + ":" +"Getting forecast scalar for %s over %s" % (rule_variation_name, ", ".join(instrument_list)))
## Get forecasts for each instrument
forecast_list=[
this_stage.get_raw_forecast(instrument_code, rule_variation_name)
for instrument_code in instrument_list]
cs_forecasts=pd.concat(forecast_list, axis=1)
scaling_factor=scalar_function(cs_forecasts, **forecast_scalar_config)
return scaling_factor
## Get some useful stuff from the config
forecast_scalar_config=copy(self.parent.config.forecast_scalar_estimate)
# The config contains 'func' and some other arguments
# we turn func which could be a string into a function, and then
# call it with the other ags
scalarfunction = resolve_function(forecast_scalar_config.pop('func'))
## this determines whether we pool or not
pool_instruments=str2Bool(forecast_scalar_config.pop("pool_instruments"))
if pool_instruments:
## pooled, same for all instruments
instrument_code_key=ALL_KEYNAME
instrument_list=self.parent.get_instrument_list()
else:
## not pooled
instrument_code_key=instrument_code
instrument_list=[instrument_code]
forecast_scalar = self.parent.calc_or_cache_nested(
"get_forecast_scalar", instrument_code_key, rule_variation_name,
_get_forecast_scalar, self, instrument_list, scalarfunction, forecast_scalar_config)
return forecast_scalar
def with_class_object(self):
class_of_entry_list = resolve_function(self.class_of_entry_list_as_str)
return classWithListOfEntriesAsListOfDicts(
class_of_entry_list, self.list_of_entries_as_list_of_dicts)
def calculation_of_pooled_raw_forecast_weights(self, instrument_code):
"""
Estimate the forecast weights for this instrument
We store this intermediate step to expose the calculation object
:param instrument_code:
:type str:
:returns: TxK pd.DataFrame containing weights, columns are trading rule variation names, T covers all
"""
def _calculation_of_pooled_raw_forecast_weights(
system, instrument_code_ref, this_stage, codes_to_use,
weighting_func, **weighting_params):
this_stage.log.terse(
"Calculating pooled raw forecast weights over instruments: %s"
% instrument_code_ref)
rule_list = self.apply_cost_weighting(instrument_code)
weight_func = weighting_func(log=self.log.setup(call="weighting"),
**weighting_params)
if weight_func.need_data():
## returns a list of accountCurveGroups
## cost pooling will already have been applied
pandl_forecasts = [
this_stage.get_returns_for_optimisation(code)
for code in codes_to_use
]
## have to decode these
## returns two lists of pd.DataFrames
(pandl_forecasts_gross,
pandl_forecasts_costs) = decompose_group_pandl(
pandl_forecasts, pool_costs=True)
## The weighting function requires two lists of pd.DataFrames, one gross, one for costs
weight_func.set_up_data(data_gross=pandl_forecasts_gross,
data_costs=pandl_forecasts_costs)
else:
## in the case of equal weights, don't need data
forecasts = this_stage.get_all_forecasts(
instrument_code, rule_list)
weight_func.set_up_data(weight_matrix=forecasts)
SR_cost_list = [
this_stage.get_SR_cost_for_instrument_forecast(
instrument_code, rule_variation_name)
for rule_variation_name in rule_list
]
weight_func.optimise(ann_SR_costs=SR_cost_list)
return weight_func
## Get some useful stuff from the config
weighting_params = copy(self.parent.config.forecast_weight_estimate)
## do we pool our estimation?
pooling_returns = str2Bool(weighting_params.pop("pool_gross_returns"))
pooling_costs = self.parent.config.forecast_cost_estimates[
'use_pooled_costs']
assert pooling_returns and pooling_costs
## which function to use for calculation
weighting_func = resolve_function(weighting_params.pop("func"))
codes_to_use = self.has_same_cheap_rules_as_code(instrument_code)
instrument_code_ref = "_".join(
codes_to_use) ## ensures we don't repeat optimisation
##
## _get_raw_forecast_weights: function to call if we don't find in cache
## self: this_system stage object
## codes_to_use: instrument codes to get data for
## weighting_func: function to call to calculate weights
## **weighting_params: parameters to pass to weighting function
##
raw_forecast_weights_calcs = self.parent.calc_or_cache(
'calculation_of_raw_forecast_weights', instrument_code_ref,
_calculation_of_pooled_raw_forecast_weights, self, codes_to_use,
weighting_func, **weighting_params)
return raw_forecast_weights_calcs
def get_forecast_correlation_matrices(self, instrument_code):
"""
Returns a correlationList object which contains a history of correlation matricies
:param instrument_code:
:type str:
:returns: correlation_list object
>>> from systems.tests.testdata import get_test_object_futures_with_rules_and_capping_estimate
>>> from systems.basesystem import System
>>> (accounts, fcs, rules, rawdata, data, config)=get_test_object_futures_with_rules_and_capping_estimate()
>>> system=System([rawdata, rules, fcs, accounts, ForecastCombineEstimated()], data, config)
>>> ans=system.combForecast.get_forecast_correlation_matrices("EDOLLAR")
>>> ans.corr_list[-1]
array([[ 1. , 0.1168699 , 0.08038547],
[ 0.1168699 , 1. , 0.86907623],
[ 0.08038547, 0.86907623, 1. ]])
>>> print(ans.columns)
['carry', 'ewmac16', 'ewmac8']
"""
def _get_forecast_correlation_matrices(system, NotUsed1, NotUsed2, this_stage,
codes_to_use, corr_func, **corr_params):
print(__file__ + ":" + str(inspect.getframeinfo(inspect.currentframe())[:3][1]) + ":" +"Calculating forecast correlations over %s" % ", ".join(codes_to_use))
forecast_data=[this_stage.get_all_forecasts(instr_code, this_stage.apply_cost_weighting(instr_code)) for instr_code in codes_to_use]
## if we're not pooling passes a list of one
forecast_data=[forecast_ts.ffill() for forecast_ts in forecast_data]
return corr_func(forecast_data, log=self.log.setup(call="correlation"), **corr_params)
## Get some useful stuff from the config
corr_params=copy(self.parent.config.forecast_correlation_estimate)
## do we pool our estimation?
pooling=str2Bool(corr_params.pop("pool_instruments"))
## which function to use for calculation
corr_func=resolve_function(corr_params.pop("func"))
if pooling:
## find set of instruments with same trading rules as I have
codes_to_use=self.has_same_cheap_rules_as_code(instrument_code)
instrument_code_ref=ALL_KEYNAME
## We
label='_'.join(codes_to_use)
else:
codes_to_use=[instrument_code]
label=instrument_code
instrument_code_ref=instrument_code
##
## label: how we identify this thing in the cache
## instrument_code_ref: eithier the instrument code, or 'all markets' if pooling
## _get_forecast_correlation_matrices: function to call if we don't find in cache
## self: this_system stage object
## codes_to_use: instrument codes
## func: function to call to calculate correlations
## **corr_params: parameters to pass to correlation function
##
forecast_corr_list = self.parent.calc_or_cache_nested(
'get_forecast_correlation_matrices', instrument_code_ref, label,
_get_forecast_correlation_matrices,
self, codes_to_use, corr_func, **corr_params)
return forecast_corr_list
def calculation_of_raw_forecast_weights_for_instrument(self, instrument_code):
"""
Does an optimisation for a single instrument
We do this if we can't do the special case of a pooled optimisation
Estimate the forecast weights for this instrument
We store this intermediate step to expose the calculation object
:param instrument_code:
:type str:
:returns: TxK pd.DataFrame containing weights, columns are trading rule variation names, T covers all
"""
def _calculation_of_raw_forecast_weights(system, instrument_code, this_stage,
codes_to_use, weighting_func, pool_costs, **weighting_params):
this_stage.log.terse("Calculating raw forecast weights for %s, over %s" % (instrument_code, ", ".join(codes_to_use)))
rule_list = self.apply_cost_weighting(instrument_code)
weight_func=weighting_func(log=self.log.setup(call="weighting"), **weighting_params)
if weight_func.need_data():
## returns a list of accountCurveGroups
pandl_forecasts=[this_stage.get_returns_for_optimisation(code)
for code in codes_to_use]
## the current curve is special
pandl_forecasts_this_code=this_stage.get_returns_for_optimisation(instrument_code)
## have to decode these
## returns two lists of pd.DataFrames
(pandl_forecasts_gross, pandl_forecasts_costs) = decompose_group_pandl(pandl_forecasts, pandl_forecasts_this_code, pool_costs=pool_costs)
## The weighting function requires two lists of pd.DataFrames, one gross, one for costs
weight_func.set_up_data(data_gross = pandl_forecasts_gross, data_costs = pandl_forecasts_costs)
else:
## in the case of equal weights, don't need data
forecasts = this_stage.get_all_forecasts(instrument_code, rule_list)
weight_func.set_up_data(weight_matrix=forecasts)
SR_cost_list = [this_stage.get_SR_cost_for_instrument_forecast(instrument_code, rule_variation_name)
for rule_variation_name in rule_list]
weight_func.optimise(ann_SR_costs=SR_cost_list)
return weight_func
## Get some useful stuff from the config
weighting_params=copy(self.parent.config.forecast_weight_estimate)
## do we pool our estimation?
pooling_returns = str2Bool(self.parent.config.forecast_weight_estimate["pool_gross_returns"])
pool_costs = str2Bool(self.parent.config.forecast_cost_estimates["use_pooled_costs"])
## which function to use for calculation
weighting_func=resolve_function(weighting_params.pop("func"))
if pooling_returns:
## find set of instruments with same trading rules as I have
codes_to_use=self.has_same_cheap_rules_as_code(instrument_code)
else:
codes_to_use=[instrument_code]
##
## _get_raw_forecast_weights: function to call if we don't find in cache
## self: this_system stage object
## codes_to_use: instrument codes to get data for
## weighting_func: function to call to calculate weights
## **weighting_params: parameters to pass to weighting function
##
raw_forecast_weights_calcs = self.parent.calc_or_cache(
'calculation_of_raw_forecast_weights', instrument_code,
_calculation_of_raw_forecast_weights,
self, codes_to_use, weighting_func, pool_costs, **weighting_params)
return raw_forecast_weights_calcs
pass
def _calculation_of_raw_forecast_weights_for_instrument(
self, instrument_code):
"""
Does an optimisation for a single instrument
We do this if we can't do the special case of a fully pooled optimisation (both costs and returns pooled)
Estimate the forecast weights for this instrument
We store this intermediate step to expose the calculation object
:param instrument_code:
:type str:
:returns: TxK pd.DataFrame containing weights, columns are trading rule variation names, T covers all
"""
# usual pool fix
# Get some useful stuff from the config
weighting_params = copy(self.parent.config.forecast_weight_estimate)
# do we pool our estimation? (note if both are pooled would better calling the function for full pooling
# eithier gross returns or costs can be pooled
pooling_returns = str2Bool(
self.parent.config.forecast_weight_estimate["pool_gross_returns"])
pool_costs = str2Bool(
self.parent.config.forecast_cost_estimates["use_pooled_costs"])
# which function to use for calculation
weighting_func = resolve_function(weighting_params.pop("func"))
# FIXME: Returns and costs are pooled in different places, very confusing
if pooling_returns:
# find set of instruments with same trading rules as I have
codes_to_use = self.has_same_cheap_rules_as_code(instrument_code)
else:
codes_to_use = [instrument_code]
self.log.terse(
"Calculating raw forecast weights for %s, over %s" %
(instrument_code, ", ".join(codes_to_use)))
rule_list = self.check_for_cheap_enough_rules(instrument_code)
# FIXME: change the way log is passed to a 'parent' style
weight_func = weighting_func(
log=self.log.setup(
call="weighting"),
**weighting_params)
# returns a list of accountCurveGroups
pandl_forecasts = [self.get_returns_for_optimisation(code)
for code in codes_to_use]
# the current curve is special
# FIXME couldn't the optimiser do this?:
pandl_forecasts_this_code = self.get_returns_for_optimisation(
instrument_code)
# have to decode these
# returns two lists of pd.DataFrames
# FIXME: WHY do this? Instead get the optimiser to do it??
(pandl_forecasts_gross, pandl_forecasts_costs) = decompose_group_pandl(
pandl_forecasts, pandl_forecasts_this_code, pool_costs=pool_costs)
# The weighting function requires two lists of pd.DataFrames,
# one gross, one for costs
weight_func.set_up_data(
data_gross=pandl_forecasts_gross,
data_costs=pandl_forecasts_costs)
weight_func.optimise()
return weight_func
def get_forecast_scalar(self, instrument_code, rule_variation_name):
"""
Get the scalar to apply to raw forecasts
If not cached, these are estimated from past forecasts
If configuration variable pool_forecasts_for_scalar is "True", then we
do this across instruments.
:param instrument_code:
:type str:
:param rule_variation_name:
:type str: name of the trading rule variation
:returns: float
>>> from systems.tests.testdata import get_test_object_futures_with_rules
>>> from systems.basesystem import System
>>> (rules, rawdata, data, config)=get_test_object_futures_with_rules()
>>> system1=System([rawdata, rules, ForecastScaleCapEstimated()], data, config)
>>>
>>> ## From default
>>> system1.forecastScaleCap.get_forecast_scalar("EDOLLAR", "ewmac8").tail(3)
scale_factor
2015-12-09 5.849888
2015-12-10 5.850474
2015-12-11 5.851091
>>> system1.forecastScaleCap.get_capped_forecast("EDOLLAR", "ewmac8").tail(3)
ewmac8
2015-12-09 0.645585
2015-12-10 -0.210377
2015-12-11 0.961821
>>>
>>> ## From config
>>> scale_config=dict(pool_instruments=False)
>>> config.forecast_scalar_estimate=scale_config
>>> system3=System([rawdata, rules, ForecastScaleCapEstimated()], data, config)
>>> system3.forecastScaleCap.get_forecast_scalar("EDOLLAR", "ewmac8").tail(3)
scale_factor
2015-12-09 5.652174
2015-12-10 5.652833
2015-12-11 5.653444
>>>
"""
def _get_forecast_scalar(system, Not_Used, rule_variation_name,
this_stage, instrument_list, scalar_function,
forecast_scalar_config):
"""
instrument_list contains multiple things, pools everything across
all instruments
"""
this_stage.log.msg(
"Getting forecast scalar for %s over %s" %
(rule_variation_name, ", ".join(instrument_list)),
rule_variation_name=rule_variation_name)
# Get forecasts for each instrument
forecast_list = [
this_stage.get_raw_forecast(instrument_code,
rule_variation_name)
for instrument_code in instrument_list
]
cs_forecasts = pd.concat(forecast_list, axis=1)
scaling_factor = scalar_function(cs_forecasts,
**forecast_scalar_config)
return scaling_factor
# Get some useful stuff from the config
forecast_scalar_config = copy(
self.parent.config.forecast_scalar_estimate)
# The config contains 'func' and some other arguments
# we turn func which could be a string into a function, and then
# call it with the other ags
scalarfunction = resolve_function(forecast_scalar_config.pop('func'))
# this determines whether we pool or not
pool_instruments = str2Bool(
forecast_scalar_config.pop("pool_instruments"))
if pool_instruments:
# pooled, same for all instruments
instrument_code_key = ALL_KEYNAME
instrument_list = self.parent.get_instrument_list()
else:
## not pooled
instrument_code_key = instrument_code
instrument_list = [instrument_code]
forecast_scalar = self.parent.calc_or_cache_nested(
"get_forecast_scalar", instrument_code_key, rule_variation_name,
_get_forecast_scalar, self, instrument_list, scalarfunction,
forecast_scalar_config)
return forecast_scalar
def calculation_of_raw_forecast_weights(self, instrument_code):
"""
Estimate the forecast weights for this instrument
We store this intermediate step to expose the calculation object
:param instrument_code:
:type str:
:returns: TxK pd.DataFrame containing weights, columns are trading rule variation names, T covers all
"""
def _calculation_of_raw_forecast_weights(system, instrument_code, this_stage,
codes_to_use, weighting_func, pool_costs=False, **weighting_params):
this_stage.log.terse("Calculating raw forecast weights over %s" % ", ".join(codes_to_use))
if hasattr(system, "accounts"):
## returns a list of accountCurveGroups
pandl_forecasts=[this_stage.pandl_for_instrument_rules_unweighted(code)
for code in codes_to_use]
## the current curve is special
pandl_forecasts_this_code=this_stage.pandl_for_instrument_rules_unweighted(instrument_code)
## have to decode these
## returns two lists of pd.DataFrames
(pandl_forecasts_gross, pandl_forecasts_costs) = decompose_group_pandl(pandl_forecasts, pandl_forecasts_this_code, pool_costs=pool_costs)
else:
error_msg="You need an accounts stage in the system to estimate forecast weights"
this_stage.log.critical(error_msg)
## The weighting function requires two lists of pd.DataFrames, one gross, one for costs
output=weighting_func(pandl_forecasts_gross, pandl_forecasts_costs,
log=self.log.setup(call="weighting"), **weighting_params)
return output
## Get some useful stuff from the config
weighting_params=copy(self.parent.config.forecast_weight_estimate)
## do we pool our estimation?
pooling=str2Bool(weighting_params.pop("pool_instruments"))
## which function to use for calculation
weighting_func=resolve_function(weighting_params.pop("func"))
if pooling:
## find set of instruments with same trading rules as I have
codes_to_use=self._has_same_rules_as_code(instrument_code)
else:
codes_to_use=[instrument_code]
##
## _get_raw_forecast_weights: function to call if we don't find in cache
## self: this_system stage object
## codes_to_use: instrument codes to get data for
## weighting_func: function to call to calculate weights
## **weighting_params: parameters to pass to weighting function
##
raw_forecast_weights_calcs = self.parent.calc_or_cache(
'calculation_of_raw_forecast_weights', instrument_code,
_calculation_of_raw_forecast_weights,
self, codes_to_use, weighting_func, **weighting_params)
return raw_forecast_weights_calcs
def get_instrument_correlation_matrix(self):
"""
Returns a correlationList object which contains a history of correlation matricies
:returns: correlation_list object
>>> from systems.tests.testdata import get_test_object_futures_with_pos_sizing_estimates
>>> from systems.basesystem import System
>>> (account, posobject, combobject, capobject, rules, rawdata, data, config)=get_test_object_futures_with_pos_sizing_estimates()
>>> system=System([rawdata, rules, posobject, combobject, capobject,PortfoliosEstimated(), account], data, config)
>>> system.config.forecast_weight_estimate["method"]="shrinkage" ## speed things up
>>> system.config.forecast_weight_estimate["date_method"]="in_sample" ## speed things up
>>> system.config.instrument_weight_estimate["date_method"]="in_sample" ## speed things up
>>> system.config.instrument_weight_estimate["method"]="shrinkage" ## speed things up
>>> ans=system.portfolio.get_instrument_correlation_matrix()
>>> ans.corr_list[-1]
array([[ 1. , 0.56981346, 0.62458477],
[ 0.56981346, 1. , 0.88087893],
[ 0.62458477, 0.88087893, 1. ]])
>>> print(ans.corr_list[0])
[[ 1. 0.99 0.99]
[ 0.99 1. 0.99]
[ 0.99 0.99 1. ]]
>>> print(ans.corr_list[10])
[[ 1. 0.99 0.99 ]
[ 0.99 1. 0.78858156]
[ 0.99 0.78858156 1. ]]
"""
def _get_instrument_correlation_matrix(system, NotUsed, this_stage,
corr_func, **corr_params):
this_stage.log.terse("Calculating instrument correlations")
instrument_codes=system.get_instrument_list()
if hasattr(system, "accounts"):
pandl=this_stage.pandl_across_subsystems().to_frame()
else:
error_msg="You need an accounts stage in the system to estimate instrument correlations"
this_stage.log.critical(error_msg)
## Need to resample here, because the correlation function won't do it properly
frequency=corr_params['frequency']
pandl=pandl.cumsum().resample(frequency).diff()
return corr_func(pandl, log=this_stage.log.setup(call="correlation"), **corr_params)
## Get some useful stuff from the config
corr_params=copy(self.parent.config.instrument_correlation_estimate)
## which function to use for calculation
corr_func=resolve_function(corr_params.pop("func"))
## _get_instrument_correlation_matrix: function to call if we don't find in cache
## self: this_system stage object
## func: function to call to calculate correlations
## **corr_params: parameters to pass to correlation function
##
forecast_corr_list = self.parent.calc_or_cache(
'get_instrument_correlation_matrix', ALL_KEYNAME,
_get_instrument_correlation_matrix,
self, corr_func, **corr_params)
return forecast_corr_list
def __init__(self, rule, data=list(), other_args=dict()):
"""
Create a trading rule from a function
Functions must be of the form function(*dataargs, **kwargs), where
*dataargs are unnamed data items, and **kwargs are named configuration
items data, an ordered list of strings identifying data to be used
(default, just price) other_args: a dictionary of named arguments to be
passed to the trading rule
:param rule: Trading rule to be created
:type trading_rules:
The following describe a rule completely (ignore data and other_args arguments)
3-tuple ; containing (function, data, other_args)
dict (containing key "function", and optionally keys "other_args" and "data")
TradingRule (object is created out of this rule)
The following will be combined with the data and other_args arguments
to produce a complete TradingRule:
Other callable function
str (with path to function eg
"systems.provide.example.rules.ewmac_forecast_with_defaults")
:param data: (list of) str pointing to location of inputs in a system method call (eg "data.get_instrument_price")
(Eithier passed in separately, or as part of a TradingRule, 3-tuple, or dict object)
:type data: single str, or list of str
:param other_args: Other named arguments to be passed to trading rule function
(Eithier passed in separately , or as part of a TradingRule, 3-tuple, or dict object)
:type other_args: dict
:returns: single Tradingrule object
"""
if hasallattr(rule, ["function", "data", "other_args"]):
# looks like it is already a trading rule
(rule_function, data, other_args) = (
rule.function, rule.data, rule.other_args)
elif isinstance(rule, tuple):
if len(data) > 0 or len(other_args) > 0:
print(
"WARNING: Creating trade rule with 'rule' tuple argument, ignoring data and/or other args")
if len(rule) != 3:
raise Exception(
"Creating trading rule with a tuple, must be length 3 exactly (function/name, data [...], args dict(...))")
(rule_function, data, other_args) = rule
elif isinstance(rule, dict):
if len(data) > 0 or len(other_args) > 0:
print(
"WARNING: Creating trade rule with 'rule' dict argument, ignoring data and/or other args")
try:
rule_function = rule['function']
except KeyError:
raise Exception(
"If you specify a TradingRule as a dict it has to contain a 'function' keyname")
if "data" in rule:
data = rule['data']
else:
data = []
if "other_args" in rule:
other_args = rule['other_args']
else:
other_args = dict()
else:
rule_function = rule
# turn string into a callable function if required
rule_function = resolve_function(rule_function)
if isinstance(data, str):
# turn into a 1 item list or wont' get parsed properly
data = [data]
setattr(self, "function", rule_function)
setattr(self, "data", data)
setattr(self, "other_args", other_args)
def calculation_of_pooled_raw_forecast_weights(self, instrument_code):
"""
Estimate the forecast weights for this instrument
We store this intermediate step to expose the calculation object
:param instrument_code:
:type str:
:returns: TxK pd.DataFrame containing weights, columns are trading rule variation names, T covers all
"""
def _calculation_of_pooled_raw_forecast_weights(system, instrument_code_ref, this_stage,
codes_to_use, weighting_func, **weighting_params):
print(__file__ + ":" + str(inspect.getframeinfo(inspect.currentframe())[:3][1]) + ":" +"Calculating pooled raw forecast weights over instruments: %s" % instrument_code_ref)
rule_list = self.apply_cost_weighting(instrument_code)
weight_func=weighting_func(log=self.log.setup(call="weighting"), **weighting_params)
if weight_func.need_data():
## returns a list of accountCurveGroups
## cost pooling will already have been applied
pandl_forecasts=[this_stage.get_returns_for_optimisation(code)
for code in codes_to_use]
## have to decode these
## returns two lists of pd.DataFrames
(pandl_forecasts_gross, pandl_forecasts_costs) = decompose_group_pandl(pandl_forecasts, pool_costs=True)
## The weighting function requires two lists of pd.DataFrames, one gross, one for costs
weight_func.set_up_data(data_gross = pandl_forecasts_gross, data_costs = pandl_forecasts_costs)
else:
## in the case of equal weights, don't need data
forecasts = this_stage.get_all_forecasts(instrument_code, rule_list)
weight_func.set_up_data(weight_matrix=forecasts)
SR_cost_list = [this_stage.get_SR_cost_for_instrument_forecast(instrument_code, rule_variation_name)
for rule_variation_name in rule_list]
weight_func.optimise(ann_SR_costs=SR_cost_list)
return weight_func
## Get some useful stuff from the config
weighting_params=copy(self.parent.config.forecast_weight_estimate)
## do we pool our estimation?
pooling_returns = str2Bool(weighting_params.pop("pool_gross_returns"))
pooling_costs = self.parent.config.forecast_cost_estimates['use_pooled_costs']
assert pooling_returns and pooling_costs
## which function to use for calculation
weighting_func=resolve_function(weighting_params.pop("func"))
codes_to_use=self.has_same_cheap_rules_as_code(instrument_code)
instrument_code_ref ="_".join(codes_to_use) ## ensures we don't repeat optimisation
##
## _get_raw_forecast_weights: function to call if we don't find in cache
## self: this_system stage object
## codes_to_use: instrument codes to get data for
## weighting_func: function to call to calculate weights
## **weighting_params: parameters to pass to weighting function
##
raw_forecast_weights_calcs = self.parent.calc_or_cache(
'calculation_of_raw_forecast_weights', instrument_code_ref,
_calculation_of_pooled_raw_forecast_weights,
self, codes_to_use, weighting_func, **weighting_params)
return raw_forecast_weights_calcs
def get_forecast_diversification_multiplier_estimated(
self, instrument_code: str) -> pd.Series:
"""
Get the diversification multiplier for this instrument
Estimated from correlations and weights
:param instrument_code: instrument to get multiplier for
:type instrument_code: str
:returns: Tx1 pd.DataFrame
>>> from systems.tests.testdata import get_test_object_futures_with_rules_and_capping_estimate
>>> from systems.basesystem import System
>>> (accounts, fcs, rules, rawdata, data, config)=get_test_object_futures_with_rules_and_capping_estimate()
>>> system=System([accounts, rawdata, rules, fcs, ForecastCombineEstimated()], data, config)
>>> system.config.forecast_weight_estimate['method']="shrinkage"
>>> system.combForecast.get_forecast_diversification_multiplier("EDOLLAR").tail(3)
FDM
2015-12-09 1.367351
2015-12-10 1.367349
2015-12-11 1.367347
>>> system.config.forecast_div_mult_estimate['dm_max']=1.1
>>> system=System([accounts, rawdata, rules, fcs, ForecastCombineEstimated()], data, system.config)
>>> system.combForecast.get_forecast_diversification_multiplier("EDOLLAR").tail(3)
FDM
2015-12-09 1.1
2015-12-10 1.1
2015-12-11 1.1
"""
self.log.terse(
"Calculating forecast div multiplier for %s" % instrument_code,
instrument_code=instrument_code,
)
# Get some useful stuff from the config
div_mult_params = copy(self.parent.config.forecast_div_mult_estimate)
# an example of an idm calculation function is
# sysquant.estimators.diversification_multipliers.diversification_multiplier_from_list
idm_func = resolve_function(div_mult_params.pop("func"))
correlation_list = self.get_forecast_correlation_matrices(
instrument_code)
## weights will be on same frequency as forecaster
weight_df = self.get_forecast_weights(instrument_code)
# note there is a possibility that the forecast_weights contain a subset of the rules in the correlation
# matrices, because the forecast weights could have rules removed for being too expensive
# To deal with this we pad the weights data frame so it is exactly
# aligned with the correlations
weight_df = dataframe_pad(weight_df,
correlation_list.column_names,
padwith=0.0)
ts_fdm = idm_func(correlation_list, weight_df, **div_mult_params)
return ts_fdm
def __init__(self, method, optimise_params, moments_estimator):
print(__file__ + ":" + str(inspect.getframeinfo(inspect.currentframe())[:3][1]) + ":" + "optimiserWithParams")
opt_func=bootstrap_portfolio
setattr(self, "opt_func", resolve_function(opt_func))
setattr(self, "params", optimise_params)
setattr(self, "moments_estimator", moments_estimator)
def _data_class(self):
class_name = self._data_class_name()
return resolve_function(class_name)
def __init__(self, method, optimise_params, moments_estimator):
print(__file__ + ":" + str(inspect.getframeinfo(inspect.currentframe())[:3][1]) + ":" + "optimiserWithParams")
opt_func=bootstrap_portfolio
setattr(self, "opt_func", resolve_function(opt_func))
setattr(self, "params", optimise_params)
setattr(self, "moments_estimator", moments_estimator)
def get_empty_series_for_timed_entry(new_entry: timedEntry) -> listOfEntries:
containing_data_class_name = new_entry.containing_data_class_name
containing_data_class = resolve_function(containing_data_class_name)
return containing_data_class.as_empty()
def create_broker_order_for_contract_order(self, contract_order_id, check_if_open=True):
original_contract_order = self.contract_stack.get_order_with_id_from_stack(contract_order_id)
log = original_contract_order.log_with_attributes(self.log)
data_locks = dataLocks(self.data)
instrument_locked = data_locks.is_instrument_locked(original_contract_order.instrument_code)
if instrument_locked:
log.msg("Instrument is locked, not spawning order")
return None
if check_if_open:
data_broker = dataBroker(self.data)
market_open = data_broker.is_instrument_code_and_contract_date_okay_to_trade(original_contract_order.instrument_code,
original_contract_order.contract_id)
if not market_open:
return None
# We can deal with partially filled contract orders: that's how hard we are!
remaining_contract_order = original_contract_order.order_with_remaining()
## Check the order doesn't breach trade limits
contract_order = self.what_contract_trade_is_possible(remaining_contract_order)
## Note we don't save the algo method, but reallocate each time
## This is useful if trading is about to finish, because we switch to market orders
## (assuming a bunch of limit orders haven't worked out so well)
contract_order = check_and_if_required_allocate_algo_to_single_contract_order(self.data, contract_order)
algo_to_use_str = contract_order.algo_to_use
algo_method = resolve_function(algo_to_use_str)
## The algo method submits an order to the broker, and returns a broker order object
## We then save the brokerorder in the broker stack, and add it as a child to a contract order
## Algos may be 'fire and forget' (a simple market order, as implemented initially) or 'active'
## Active algos need to keep running on another thread (need to work out how to do this)
## They will set the property 'reference_of_controlling_algo' in contract order
## Fills are picked up by another process (or if the algo is an active thing, potentially by itself)
broker_order, reference_of_controlling_algo = algo_method(self.data, contract_order)
if broker_order is missing_order:
# something bad has happened and we can't submit an order to the broker
# Nae bother, maybe try again later
# Unlock the contract order in case we want to do this later
self.contract_stack.release_order_from_algo_control(contract_order_id)
return None
## update trade limits
self.add_trade_to_trade_limits(broker_order)
broker_order_id = self.broker_stack.put_order_on_stack(broker_order)
if type(broker_order_id) is not int:
# We've created a broker order but can't add it to the broker order database
# Probably safest to leave the contract order locked otherwise there could be multiple
# broker orders issued and nobody wants that!
log.critical("Created a broker order %s but can't add it to the order stack!! (condition %s)" %
(str(broker_order), str(broker_order_id)))
return failure
# ....create new algo lock
# This means nobody else can try and execute this order until it is released
# Only the algo itself can release!
# This only applies to 'fire and forget' orders that aren't controlled by an algo
self.contract_stack.add_controlling_algo_ref(contract_order_id, reference_of_controlling_algo)
# This broker order is a child of the parent contract order
# We add 'another' child since it's valid to have multiple broker orders
self.contract_stack.add_another_child_to_order(contract_order_id, broker_order_id)
return success
def calculation_of_raw_forecast_weights(self, instrument_code):
"""
Estimate the forecast weights for this instrument
We store this intermediate step to expose the calculation object
:param instrument_code:
:type str:
:returns: TxK pd.DataFrame containing weights, columns are trading rule variation names, T covers all
"""
def _calculation_of_raw_forecast_weights(system,
instrument_code,
this_stage,
codes_to_use,
weighting_func,
pool_costs=False,
**weighting_params):
this_stage.log.terse("Calculating raw forecast weights over %s" %
", ".join(codes_to_use))
if hasattr(system, "accounts"):
## returns a list of accountCurveGroups
pandl_forecasts = [
this_stage.pandl_for_instrument_rules_unweighted(code)
for code in codes_to_use
]
## the current curve is special
pandl_forecasts_this_code = this_stage.pandl_for_instrument_rules_unweighted(
instrument_code)
## have to decode these
## returns two lists of pd.DataFrames
(pandl_forecasts_gross,
pandl_forecasts_costs) = decompose_group_pandl(
pandl_forecasts,
pandl_forecasts_this_code,
pool_costs=pool_costs)
else:
error_msg = "You need an accounts stage in the system to estimate forecast weights"
this_stage.log.critical(error_msg)
## The weighting function requires two lists of pd.DataFrames, one gross, one for costs
output = weighting_func(pandl_forecasts_gross,
pandl_forecasts_costs,
log=self.log.setup(call="weighting"),
**weighting_params)
return output
## Get some useful stuff from the config
weighting_params = copy(self.parent.config.forecast_weight_estimate)
## do we pool our estimation?
pooling = str2Bool(weighting_params.pop("pool_instruments"))
## which function to use for calculation
weighting_func = resolve_function(weighting_params.pop("func"))
if pooling:
## find set of instruments with same trading rules as I have
codes_to_use = self._has_same_rules_as_code(instrument_code)
else:
codes_to_use = [instrument_code]
##
## _get_raw_forecast_weights: function to call if we don't find in cache
## self: this_system stage object
## codes_to_use: instrument codes to get data for
## weighting_func: function to call to calculate weights
## **weighting_params: parameters to pass to weighting function
##
raw_forecast_weights_calcs = self.parent.calc_or_cache(
'calculation_of_raw_forecast_weights', instrument_code,
_calculation_of_raw_forecast_weights, self, codes_to_use,
weighting_func, **weighting_params)
return raw_forecast_weights_calcs
def get_forecast_correlation_matrices(self, instrument_code):
"""
Returns a correlationList object which contains a history of correlation matricies
:param instrument_code:
:type str:
:returns: correlation_list object
>>> from systems.tests.testdata import get_test_object_futures_with_rules_and_capping_estimate
>>> from systems.basesystem import System
>>> (accounts, fcs, rules, rawdata, data, config)=get_test_object_futures_with_rules_and_capping_estimate()
>>> system=System([rawdata, rules, fcs, accounts, ForecastCombineEstimated()], data, config)
>>> ans=system.combForecast.get_forecast_correlation_matrices("EDOLLAR")
>>> ans.corr_list[-1]
array([[ 1. , 0.1168699 , 0.08038547],
[ 0.1168699 , 1. , 0.86907623],
[ 0.08038547, 0.86907623, 1. ]])
>>> print(ans.columns)
['carry', 'ewmac16', 'ewmac8']
"""
def _get_forecast_correlation_matrices(system, NotUsed1, NotUsed2,
this_stage, codes_to_use,
corr_func, **corr_params):
this_stage.log.terse("Calculating forecast correlations over %s" %
", ".join(codes_to_use))
forecast_data = [
this_stage.get_all_forecasts(
instr_code, this_stage.apply_cost_weighting(instr_code))
for instr_code in codes_to_use
]
## if we're not pooling passes a list of one
forecast_data = [
forecast_ts.ffill() for forecast_ts in forecast_data
]
return corr_func(forecast_data,
log=self.log.setup(call="correlation"),
**corr_params)
## Get some useful stuff from the config
corr_params = copy(self.parent.config.forecast_correlation_estimate)
## do we pool our estimation?
pooling = str2Bool(corr_params.pop("pool_instruments"))
## which function to use for calculation
corr_func = resolve_function(corr_params.pop("func"))
if pooling:
## find set of instruments with same trading rules as I have
codes_to_use = self.has_same_cheap_rules_as_code(instrument_code)
instrument_code_ref = ALL_KEYNAME
## We
label = '_'.join(codes_to_use)
else:
codes_to_use = [instrument_code]
label = instrument_code
instrument_code_ref = instrument_code
##
## label: how we identify this thing in the cache
## instrument_code_ref: eithier the instrument code, or 'all markets' if pooling
## _get_forecast_correlation_matrices: function to call if we don't find in cache
## self: this_system stage object
## codes_to_use: instrument codes
## func: function to call to calculate correlations
## **corr_params: parameters to pass to correlation function
##
forecast_corr_list = self.parent.calc_or_cache_nested(
'get_forecast_correlation_matrices', instrument_code_ref, label,
_get_forecast_correlation_matrices, self, codes_to_use, corr_func,
**corr_params)
return forecast_corr_list
