def __init__(self, optimise_params): """ Create an object which estimates the moments for a single period of data, according to the parameters The parameters we need are popped from the config dict :param optimise_params: Parameters for optimisation :type optimise_params: dict """ corr_estimate_params=copy(optimise_params["correlation_estimate"]) mean_estimate_params=copy(optimise_params["mean_estimate"]) vol_estimate_params=copy(optimise_params["vol_estimate"]) corr_estimate_func=resolve_function(corr_estimate_params.pop("func")) mean_estimate_func=resolve_function(mean_estimate_params.pop("func")) vol_estimate_func=resolve_function(vol_estimate_params.pop("func")) setattr(self, "corr_estimate_params", corr_estimate_params) setattr(self, "mean_estimate_params", mean_estimate_params) setattr(self, "vol_estimate_params", vol_estimate_params) setattr(self, "corr_estimate_func", corr_estimate_func) setattr(self, "mean_estimate_func", mean_estimate_func) setattr(self, "vol_estimate_func", vol_estimate_func)
def __init__(self, optimise_params, annualisation=BUSINESS_DAYS_IN_YEAR, ann_target_SR=.5): """ Create an object which estimates the moments for a single period of data, according to the parameters The parameters we need are popped from the config dict :param optimise_params: Parameters for optimisation :type optimise_params: dict """ corr_estimate_params = copy(optimise_params["correlation_estimate"]) mean_estimate_params = copy(optimise_params["mean_estimate"]) vol_estimate_params = copy(optimise_params["vol_estimate"]) corr_estimate_func = resolve_function(corr_estimate_params.pop("func")) mean_estimate_func = resolve_function(mean_estimate_params.pop("func")) vol_estimate_func = resolve_function(vol_estimate_params.pop("func")) setattr(self, "corr_estimate_params", corr_estimate_params) setattr(self, "mean_estimate_params", mean_estimate_params) setattr(self, "vol_estimate_params", vol_estimate_params) setattr(self, "corr_estimate_func", corr_estimate_func) setattr(self, "mean_estimate_func", mean_estimate_func) setattr(self, "vol_estimate_func", vol_estimate_func) period_target_SR = ann_target_SR / (annualisation**.5) setattr(self, "annualisation", annualisation) setattr(self, "period_target_SR", period_target_SR) setattr(self, "ann_target_SR", ann_target_SR)
def __init__(self, method, optimise_params, moments_estimator): """ Create an object which does an optimisation for a single period, according to the parameters :param optimise_params: Parameters for optimisation. Must contain "method" :type optimise_params: dict :param moments_estimator: An instance of a moments estimator :type optimise_params: momentsEstimator """ fit_method_dict = dict( one_period=markosolver, bootstrap=bootstrap_portfolio, shrinkage=opt_shrinkage, equal_weights=equal_weights) try: opt_func = fit_method_dict[method] except KeyError: raise Exception("Fitting method %s unknown; try one of: %s " % (method, ", ".join(fit_method_dict.keys()))) setattr(self, "opt_func", resolve_function(opt_func)) setattr(self, "params", optimise_params) setattr(self, "moments_estimator", moments_estimator)
def calculation_of_raw_instrument_weights(self): """ Estimate the instrument weights Done like this to expose calculations :returns: TxK pd.DataFrame containing weights, columns are instrument names, T covers all """ def _calculation_of_raw_instrument_weights(system, NotUsed1, this_stage, weighting_func, **weighting_params): this_stage.log.terse("Calculating raw instrument weights") instrument_codes = system.get_instrument_list() weight_func = weighting_func( log=this_stage.log.setup( call="weighting"), **weighting_params) if weight_func.need_data(): if hasattr(system, "accounts"): pandl = this_stage.pandl_across_subsystems() (pandl_gross, pandl_costs) = decompose_group_pandl([pandl]) weight_func.set_up_data( data_gross=pandl_gross, data_costs=pandl_costs) else: error_msg = "You need an accounts stage in the system to estimate instrument weights" this_stage.log.critical(error_msg) else: # equal weights doesn't need data positions = this_stage._get_all_subsystem_positions() weight_func.set_up_data(weight_matrix=positions) SR_cost_list = [this_stage.get_instrument_subsystem_SR_cost( instr_code) for instr_code in instrument_codes] weight_func.optimise(ann_SR_costs=SR_cost_list) return weight_func # Get some useful stuff from the config weighting_params = copy(self.parent.config.instrument_weight_estimate) # which function to use for calculation weighting_func = resolve_function(weighting_params.pop("func")) calcs_of_instrument_weights = self.parent.calc_or_cache( 'calculation_of_raw_instrument_weights', ALL_KEYNAME, _calculation_of_raw_instrument_weights, self, weighting_func, **weighting_params) return calcs_of_instrument_weights
def daily_returns_volatility(self, instrument_code): """ Gets volatility of daily returns (not % returns) This is done using a user defined function We get this from: the configuration object or if not found, system.defaults.py The dict must contain func key; anything else is optional :param instrument_code: Instrument to get prices for :type trading_rules: str :returns: Tx1 pd.DataFrame >>> from systems.tests.testdata import get_test_object >>> from systems.basesystem import System >>> >>> (rawdata, data, config)=get_test_object() >>> system=System([rawdata], data) >>> ## uses defaults >>> system.rawdata.daily_returns_volatility("EDOLLAR").tail(2) vol 2015-12-10 0.054145 2015-12-11 0.058522 >>> >>> from sysdata.configdata import Config >>> config=Config("systems.provided.example.exampleconfig.yaml") >>> system=System([rawdata], data, config) >>> system.rawdata.daily_returns_volatility("EDOLLAR").tail(2) vol 2015-12-10 0.054145 2015-12-11 0.058522 >>> >>> config=Config(dict(volatility_calculation=dict(func="syscore.algos.robust_vol_calc", days=200))) >>> system2=System([rawdata], data, config) >>> system2.rawdata.daily_returns_volatility("EDOLLAR").tail(2) vol 2015-12-10 0.057946 2015-12-11 0.058626 """ self.log.msg( "Calculating daily volatility for %s" % instrument_code, instrument_code=instrument_code) system = self.parent dailyreturns = self.daily_returns(instrument_code) volconfig = copy(system.config.volatility_calculation) # volconfig contains 'func' and some other arguments # we turn func which could be a string into a function, and then # call it with the other ags volfunction = resolve_function(volconfig.pop('func')) vol = volfunction(dailyreturns, **volconfig) return vol
def _get_forecast_scalar_estimated_from_instrument_list( self, instrument_code, rule_variation_name, forecast_scalar_config): """ Get the scalar to apply to raw forecasts If not cached, these are estimated from past forecasts :param instrument_code: instrument code, or ALL_KEYNAME if pooling :type str: :param rule_variation_name: :type str: name of the trading rule variation :param forecast_scalar_config: :type dict: relevant part of the config :returns: float """ # The config contains 'func' and some other arguments # we turn func which could be a string into a function, and then # call it with the other ags scalar_function = resolve_function(forecast_scalar_config.pop('func')) """ instrument_list contains multiple things, might pool everything across all instruments """ if instrument_code == ALL_KEYNAME: # pooled, same for all instruments instrument_list = self.parent.get_instrument_list() else: ## not pooled instrument_list = [instrument_code] self.log.msg( "Getting forecast scalar for %s over %s" % (rule_variation_name, ", ".join(instrument_list)), rule_variation_name=rule_variation_name) # Get forecasts for each instrument forecast_list = [ self.get_raw_forecast(instrument_code, rule_variation_name) for instrument_code in instrument_list ] cs_forecasts = pd.concat(forecast_list, axis=1) # an example of a scaling function is syscore.algos.forecast_scalar # must return thing the same size as cs_forecasts scaling_factor = scalar_function(cs_forecasts, **forecast_scalar_config) return scaling_factor
def __init__(self, optimise_params, annualisation=BUSINESS_DAYS_IN_YEAR, ann_target_SR=.5): corr_estimate_params=copy(optimise_params["correlation_estimate"]) mean_estimate_params=copy(optimise_params["mean_estimate"]) vol_estimate_params=copy(optimise_params["vol_estimate"]) corr_estimate_func=resolve_function(corr_estimate_params.pop("func")) mean_estimate_func=resolve_function(mean_estimate_params.pop("func")) vol_estimate_func=resolve_function(vol_estimate_params.pop("func")) setattr(self, "corr_estimate_params", corr_estimate_params) setattr(self, "mean_estimate_params", mean_estimate_params) setattr(self, "vol_estimate_params", vol_estimate_params) setattr(self, "corr_estimate_func", corr_estimate_func) setattr(self, "mean_estimate_func", mean_estimate_func) setattr(self, "vol_estimate_func", vol_estimate_func) period_target_SR = ann_target_SR / (annualisation**.5) setattr(self, "annualisation", annualisation) setattr(self, "period_target_SR", period_target_SR) setattr(self, "ann_target_SR", ann_target_SR)
def get_instrument_correlation_matrix(self): """ Returns a correlationList object which contains a history of correlation matricies :returns: correlation_list object >>> from systems.tests.testdata import get_test_object_futures_with_pos_sizing_estimates >>> from systems.basesystem import System >>> (account, posobject, combobject, capobject, rules, rawdata, data, config)=get_test_object_futures_with_pos_sizing_estimates() >>> system=System([rawdata, rules, posobject, combobject, capobject,PortfoliosEstimated(), account], data, config) >>> system.config.forecast_weight_estimate["method"]="shrinkage" ## speed things up >>> system.config.forecast_weight_estimate["date_method"]="in_sample" ## speed things up >>> system.config.instrument_weight_estimate["date_method"]="in_sample" ## speed things up >>> system.config.instrument_weight_estimate["method"]="shrinkage" ## speed things up >>> ans=system.portfolio.get_instrument_correlation_matrix() >>> ans.corr_list[-1] array([[ 1. , 0.56981346, 0.62458477], [ 0.56981346, 1. , 0.88087893], [ 0.62458477, 0.88087893, 1. ]]) >>> print(ans.corr_list[0]) [[ 1. 0.99 0.99] [ 0.99 1. 0.99] [ 0.99 0.99 1. ]] >>> print(ans.corr_list[10]) [[ 1. 0.99 0.99 ] [ 0.99 1. 0.78858156] [ 0.99 0.78858156 1. ]] """ self.log.terse("Calculating instrument correlations") system = self.parent # Get some useful stuff from the config corr_params = copy(system.config.instrument_correlation_estimate) # which function to use for calculation corr_func = resolve_function(corr_params.pop("func")) if hasattr(system, "accounts"): pandl = self.pandl_across_subsystems().to_frame() else: error_msg = "You need an accounts stage in the system to estimate instrument correlations" self.log.critical(error_msg) # Need to resample here, because the correlation function won't do # it properly (doesn't know it's dealing with returns data) frequency = corr_params['frequency'] pandl = pandl.cumsum().resample(frequency).last().diff() # The subsequent resample inside the correlation function will have no effect return corr_func(pandl, **corr_params)
def _daily_returns_volatility(system, instrument_code, this_stage): this_stage.log.msg("Calculating daily volatility for %s" % instrument_code, instrument_code=instrument_code) dailyreturns = this_stage.daily_returns(instrument_code) volconfig=copy(system.config.volatility_calculation) # volconfig contains 'func' and some other arguments # we turn func which could be a string into a function, and then # call it with the other ags volfunction = resolve_function(volconfig.pop('func')) vol = volfunction(dailyreturns, **volconfig) return vol
def _daily_returns_volatility(system, instrument_code, this_stage): print(__file__ + ":" + str(inspect.getframeinfo(inspect.currentframe())[:3][1]) + ":" +"Calculating daily volatility for %s" % instrument_code) dailyreturns = this_stage.daily_returns(instrument_code) volconfig=copy(system.config.volatility_calculation) # volconfig contains 'func' and some other arguments # we turn func which could be a string into a function, and then # call it with the other ags volfunction = resolve_function(volconfig.pop('func')) vol = volfunction(dailyreturns, **volconfig) return vol
def _get_forecast_div_multiplier(system, instrument_code, this_stage): print(__file__ + ":" + str(inspect.getframeinfo(inspect.currentframe())[:3][1]) + ":" +"Calculating forecast div multiplier for %s" % instrument_code) ## Get some useful stuff from the config div_mult_params=copy(system.config.forecast_div_mult_estimate) idm_func=resolve_function(div_mult_params.pop("func")) correlation_list_object=this_stage.get_forecast_correlation_matrices(instrument_code) weight_df=this_stage.get_forecast_weights(instrument_code) ts_fdm=idm_func(correlation_list_object, weight_df, **div_mult_params) return ts_fdm
def _get_instrument_div_multiplier(system, NotUsed, this_stage): this_stage.log.terse("Calculating instrument div. multiplier") ## Get some useful stuff from the config div_mult_params=copy(system.config.instrument_div_mult_estimate) idm_func=resolve_function(div_mult_params.pop("func")) correlation_list_object=this_stage.get_instrument_correlation_matrix() weight_df=this_stage.get_instrument_weights() ts_idm=idm_func(correlation_list_object, weight_df, **div_mult_params) return ts_idm
def _get_forecast_div_multiplier(system, instrument_code, this_stage): this_stage.log.terse("Calculating forecast div multiplier for %s" % instrument_code, instrument_code=instrument_code) ## Get some useful stuff from the config div_mult_params=copy(system.config.forecast_div_mult_estimate) idm_func=resolve_function(div_mult_params.pop("func")) correlation_list_object=this_stage.get_forecast_correlation_matrices(instrument_code) weight_df=this_stage.get_forecast_weights(instrument_code) ts_fdm=idm_func(correlation_list_object, weight_df, **div_mult_params) return ts_fdm
def calculation_of_raw_instrument_weights(self): """ Estimate the instrument weights Done like this to expose calculations :returns: TxK pd.DataFrame containing weights, columns are instrument names, T covers all """ def _calculation_of_raw_instrument_weights(system, NotUsed1, this_stage, weighting_func, **weighting_params): this_stage.log.terse("Calculating raw instrument weights") instrument_codes=system.get_instrument_list() if hasattr(system, "accounts"): pandl_subsystems=[this_stage.pandl_for_subsystem(code) for code in instrument_codes] else: error_msg="You need an accounts stage in the system to estimate instrument weights" this_stage.log.critical(error_msg) pandl=pd.concat(pandl_subsystems, axis=1) pandl.columns=instrument_codes instrument_weight_results=weighting_func(pandl, log=self.log.setup(call="weighting"), **weighting_params) return instrument_weight_results ## Get some useful stuff from the config weighting_params=copy(self.parent.config.instrument_weight_estimate) ## which function to use for calculation weighting_func=resolve_function(weighting_params.pop("func")) calcs_of_instrument_weights = self.parent.calc_or_cache( 'calculation_of_raw_instrument_weights', ALL_KEYNAME, _calculation_of_raw_instrument_weights, self, weighting_func, **weighting_params) return calcs_of_instrument_weights
def capital_multiplier(self, delayfill=True, roundpositions=False): """ Get a capital multiplier :param delayfill: Lag fills by one day :type delayfill: bool :param roundpositions: Round positions to whole contracts :type roundpositions: bool :returns: pd.Series """ system = self.parent capmult_params = copy(system.config.capital_multiplier) capmult_func = resolve_function(capmult_params.pop("func")) capmult = capmult_func(system, **capmult_params) capmult = capmult.reindex(self.portfolio().index).ffill() return capmult
def _daily_returns_volatility(system, instrument_code, this_stage): dailyreturns = this_stage.daily_returns(instrument_code) try: volconfig = copy(system.config.volatility_calculation) identify_error = "inherited from config object" except: volconfig = copy(system_defaults['volatility_calculation']) identify_error = "found in system.defaults.py" if "func" not in volconfig: raise Exception( "The volconfig dict (%s) needs to have a 'func' key" % identify_error) # volconfig contains 'func' and some other arguments # we turn func which could be a string into a function, and then # call it with the other ags volfunction = resolve_function(volconfig.pop('func')) vol = volfunction(dailyreturns, **volconfig) return vol
def get_estimated_instrument_diversification_multiplier(self): """ Estimate the diversification multiplier for the portfolio Estimated from correlations and weights :returns: Tx1 pd.DataFrame >>> from systems.tests.testdata import get_test_object_futures_with_pos_sizing_estimates >>> from systems.basesystem import System >>> (account, posobject, combobject, capobject, rules, rawdata, data, config)=get_test_object_futures_with_pos_sizing_estimates() >>> system=System([rawdata, rules, posobject, combobject, capobject,PortfoliosEstimated(), account], data, config) >>> system.config.forecast_weight_estimate["method"]="shrinkage" ## speed things up >>> system.config.forecast_weight_estimate["date_method"]="in_sample" ## speed things up >>> system.config.instrument_weight_estimate["date_method"]="in_sample" ## speed things up >>> system.config.instrument_weight_estimate["method"]="shrinkage" ## speed things up >>> system.portfolio.get_instrument_diversification_multiplier().tail(3) IDM 2015-12-09 1.133220 2015-12-10 1.133186 2015-12-11 1.133153 """ self.log.terse("Calculating instrument div. multiplier") # Get some useful stuff from the config div_mult_params = copy(self.parent.config.instrument_div_mult_estimate) idm_func = resolve_function(div_mult_params.pop("func")) correlation_list_object = self.get_instrument_correlation_matrix() weight_df = self.get_instrument_weights() ts_idm = idm_func(correlation_list_object, weight_df, **div_mult_params) return ts_idm
def calculation_of_raw_instrument_weights(self): """ Estimate the instrument weights Done like this to expose calculations :returns: TxK pd.DataFrame containing weights, columns are instrument names, T covers all """ # Get some useful stuff from the config weighting_params = copy(self.parent.config.instrument_weight_estimate) # which function to use for calculation weighting_func = resolve_function(weighting_params.pop("func")) system = self.parent self.log.terse("Calculating raw instrument weights") instrument_codes = system.get_instrument_list() if hasattr(system, "accounts"): pandl = self.pandl_across_subsystems() else: error_msg = "You need an accounts stage in the system to estimate instrument weights" self.log.critical(error_msg) # The optimiser is set up for pooling, but we're not going to do that # Create a single fake set of return data data = dict(instrument_pandl = pandl) weight_func = weighting_func(data, identifier ="instrument_pandl", parent=self, **weighting_params) weight_func.optimise() return weight_func
def get_instrument_correlation_matrix(self): """ Returns a correlationList object which contains a history of correlation matricies :returns: correlation_list object >>> from systems.tests.testdata import get_test_object_futures_with_pos_sizing_estimates >>> from systems.basesystem import System >>> (account, posobject, combobject, capobject, rules, rawdata, data, config)=get_test_object_futures_with_pos_sizing_estimates() >>> system=System([rawdata, rules, posobject, combobject, capobject,PortfoliosEstimated(), account], data, config) >>> system.config.forecast_weight_estimate["method"]="shrinkage" ## speed things up >>> system.config.forecast_weight_estimate["date_method"]="in_sample" ## speed things up >>> system.config.instrument_weight_estimate["date_method"]="in_sample" ## speed things up >>> system.config.instrument_weight_estimate["method"]="shrinkage" ## speed things up >>> ans=system.portfolio.get_instrument_correlation_matrix() >>> ans.corr_list[-1] array([[ 1. , 0.50156603, 0.56866684], [ 0.50156603, 1. , 0.88358678], [ 0.56866684, 0.88358678, 1. ]]) >>> print(ans.corr_list[0]) [[ 1. 0.99 0.99] [ 0.99 1. 0.99] [ 0.99 0.99 1. ]] >>> print(ans.corr_list[10]) [[ 1. 0.99 0.99 ] [ 0.99 1. 0.76599709] [ 0.99 0.76599709 1. ]] """ def _get_instrument_correlation_matrix(system, NotUsed, this_stage, corr_func, **corr_params): this_stage.log.terse("Calculating instrument correlations") instrument_codes=system.get_instrument_list() if hasattr(system, "accounts"): pandl=this_stage.pandl_across_subsystems() else: error_msg="You need an accounts stage in the system to estimate instrument correlations" this_stage.log.critical(error_msg) ## Need to resample here, because the correlation function won't do it properly frequency=corr_params['frequency'] pandl=pandl.cumsum().resample(frequency).diff() return corr_func(pandl, log=this_stage.log.setup(call="correlation"), **corr_params) ## Get some useful stuff from the config corr_params=copy(self.parent.config.instrument_correlation_estimate) ## which function to use for calculation corr_func=resolve_function(corr_params.pop("func")) ## _get_instrument_correlation_matrix: function to call if we don't find in cache ## self: this_system stage object ## func: function to call to calculate correlations ## **corr_params: parameters to pass to correlation function ## forecast_corr_list = self.parent.calc_or_cache( 'get_instrument_correlation_matrix', ALL_KEYNAME, _get_instrument_correlation_matrix, self, corr_func, **corr_params) return forecast_corr_list
def calculation_of_raw_forecast_weights(self, instrument_code): """ Estimate the forecast weights for this instrument We store this intermediate step to expose the calculation object :param instrument_code: :type str: :returns: TxK pd.DataFrame containing weights, columns are trading rule variation names, T covers all """ def _calculation_of_raw_forecast_weights(system, NotUsed1, NotUsed2, this_stage, codes_to_use, weighting_func, **weighting_params): this_stage.log.terse("Calculating raw forecast weights over %s" % ", ".join(codes_to_use)) if hasattr(system, "accounts"): pandl_forecasts=[this_stage.pandl_for_instrument_rules_unweighted(code) for code in codes_to_use] else: error_msg="You need an accounts stage in the system to estimate forecast weights" this_stage.log.critical(error_msg) output=weighting_func(pandl_forecasts, log=self.log.setup(call="weighting"), **weighting_params) return output ## Get some useful stuff from the config weighting_params=copy(self.parent.config.forecast_weight_estimate) ## do we pool our estimation? pooling=str2Bool(weighting_params.pop("pool_instruments")) ## which function to use for calculation weighting_func=resolve_function(weighting_params.pop("func")) if pooling: ## find set of instruments with same trading rules as I have codes_to_use=self._has_same_rules_as_code(instrument_code) instrument_code_ref=ALL_KEYNAME ## We label='_'.join(codes_to_use) else: codes_to_use=[instrument_code] label=instrument_code instrument_code_ref=instrument_code ## ## label: how we identify this thing in the cache ## instrument_code_ref: eithier the instrument code, or 'all markets' if pooling ## _get_raw_forecast_weights: function to call if we don't find in cache ## self: this_system stage object ## codes_to_use: instrument codes to get data for ## weighting_func: function to call to calculate weights ## **weighting_params: parameters to pass to weighting function ## raw_forecast_weights_calcs = self.parent.calc_or_cache_nested( 'calculation_of_raw_forecast_weights', instrument_code_ref, label, _calculation_of_raw_forecast_weights, self, codes_to_use, weighting_func, **weighting_params) return raw_forecast_weights_calcs
def __init__(self, rule, data=list(), other_args=dict()): """ Create a trading rule from a function Functions must be of the form function(*dataargs, **kwargs), where *dataargs are unnamed data items, and **kwargs are named configuration items data, an ordered list of strings identifying data to be used (default, just price) other_args: a dictionary of named arguments to be passed to the trading rule :param rule: Trading rule to be created :type trading_rules: The following describe a rule completely (ignore data and other_args arguments) 3-tuple ; containing (function, data, other_args) dict (containing key "function", and optionally keys "other_args" and "data") TradingRule (object is created out of this rule) The following will be combined with the data and other_args arguments to produce a complete TradingRule: Other callable function str (with path to function eg "systems.provide.example.rules.ewmac_forecast_with_defaults") :param data: (list of) str pointing to location of inputs in a system method call (eg "data.get_instrument_price") (Either passed in separately, or as part of a TradingRule, 3-tuple, or dict object) :type data: single str, or list of str :param other_args: Other named arguments to be passed to trading rule function (Either passed in separately , or as part of a TradingRule, 3-tuple, or dict object) :type other_args: dict :returns: single Tradingrule object """ if hasallattr(rule, ["function", "data", "other_args"]): # looks like it is already a trading rule (rule_function, data, other_args) = (rule.function, rule.data, rule.other_args) elif isinstance(rule, tuple): if len(data) > 0 or len(other_args) > 0: print( "WARNING: Creating trade rule with 'rule' tuple argument, ignoring data and/or other args" ) if len(rule) != 3: raise Exception( "Creating trading rule with a tuple, must be length 3 exactly (function/name, data [...], args dict(...))" ) (rule_function, data, other_args) = rule elif isinstance(rule, dict): if len(data) > 0 or len(other_args) > 0: print( "WARNING: Creating trade rule with 'rule' dict argument, ignoring data and/or other args" ) try: rule_function = rule['function'] except KeyError: raise Exception( "If you specify a TradingRule as a dict it has to contain a 'function' keyname" ) if "data" in rule: data = rule['data'] else: data = [] if "other_args" in rule: other_args = rule['other_args'] else: other_args = dict() else: rule_function = rule # turn string into a callable function if required rule_function = resolve_function(rule_function) if isinstance(data, str): # turn into a 1 item list or wont' get parsed properly data = [data] setattr(self, "function", rule_function) setattr(self, "data", data) setattr(self, "other_args", other_args)
def calculation_of_raw_forecast_weights_for_instrument( self, instrument_code): """ Does an optimisation for a single instrument We do this if we can't do the special case of a pooled optimisation Estimate the forecast weights for this instrument We store this intermediate step to expose the calculation object :param instrument_code: :type str: :returns: TxK pd.DataFrame containing weights, columns are trading rule variation names, T covers all """ def _calculation_of_raw_forecast_weights(system, instrument_code, this_stage, codes_to_use, weighting_func, pool_costs, **weighting_params): this_stage.log.terse( "Calculating raw forecast weights for %s, over %s" % (instrument_code, ", ".join(codes_to_use))) rule_list = self.apply_cost_weighting(instrument_code) weight_func = weighting_func(log=self.log.setup(call="weighting"), **weighting_params) if weight_func.need_data(): ## returns a list of accountCurveGroups pandl_forecasts = [ this_stage.get_returns_for_optimisation(code) for code in codes_to_use ] ## the current curve is special pandl_forecasts_this_code = this_stage.get_returns_for_optimisation( instrument_code) ## have to decode these ## returns two lists of pd.DataFrames (pandl_forecasts_gross, pandl_forecasts_costs) = decompose_group_pandl( pandl_forecasts, pandl_forecasts_this_code, pool_costs=pool_costs) ## The weighting function requires two lists of pd.DataFrames, one gross, one for costs weight_func.set_up_data(data_gross=pandl_forecasts_gross, data_costs=pandl_forecasts_costs) else: ## in the case of equal weights, don't need data forecasts = this_stage.get_all_forecasts( instrument_code, rule_list) weight_func.set_up_data(weight_matrix=forecasts) SR_cost_list = [ this_stage.get_SR_cost_for_instrument_forecast( instrument_code, rule_variation_name) for rule_variation_name in rule_list ] weight_func.optimise(ann_SR_costs=SR_cost_list) return weight_func ## Get some useful stuff from the config weighting_params = copy(self.parent.config.forecast_weight_estimate) ## do we pool our estimation? pooling_returns = str2Bool( self.parent.config.forecast_weight_estimate["pool_gross_returns"]) pool_costs = str2Bool( self.parent.config.forecast_cost_estimates["use_pooled_costs"]) ## which function to use for calculation weighting_func = resolve_function(weighting_params.pop("func")) if pooling_returns: ## find set of instruments with same trading rules as I have codes_to_use = self.has_same_cheap_rules_as_code(instrument_code) else: codes_to_use = [instrument_code] ## ## _get_raw_forecast_weights: function to call if we don't find in cache ## self: this_system stage object ## codes_to_use: instrument codes to get data for ## weighting_func: function to call to calculate weights ## **weighting_params: parameters to pass to weighting function ## raw_forecast_weights_calcs = self.parent.calc_or_cache( 'calculation_of_raw_forecast_weights', instrument_code, _calculation_of_raw_forecast_weights, self, codes_to_use, weighting_func, pool_costs, **weighting_params) return raw_forecast_weights_calcs pass
def get_forecast_scalar(self, instrument_code, rule_variation_name): """ Get the scalar to apply to raw forecasts If not cached, these are estimated from past forecasts If configuration variable pool_forecasts_for_scalar is "True", then we do this across instruments. :param instrument_code: :type str: :param rule_variation_name: :type str: name of the trading rule variation :returns: float >>> from systems.tests.testdata import get_test_object_futures_with_rules >>> from systems.basesystem import System >>> (rules, rawdata, data, config)=get_test_object_futures_with_rules() >>> system1=System([rawdata, rules, ForecastScaleCapEstimated()], data, config) >>> >>> ## From default >>> system1.forecastScaleCap.get_forecast_scalar("EDOLLAR", "ewmac8").tail(3) scale_factor 2015-12-09 5.849888 2015-12-10 5.850474 2015-12-11 5.851091 >>> system1.forecastScaleCap.get_capped_forecast("EDOLLAR", "ewmac8").tail(3) ewmac8 2015-12-09 0.645585 2015-12-10 -0.210377 2015-12-11 0.961821 >>> >>> ## From config >>> scale_config=dict(pool_instruments=False) >>> config.forecast_scalar_estimate=scale_config >>> system3=System([rawdata, rules, ForecastScaleCapEstimated()], data, config) >>> system3.forecastScaleCap.get_forecast_scalar("EDOLLAR", "ewmac8").tail(3) scale_factor 2015-12-09 5.652174 2015-12-10 5.652833 2015-12-11 5.653444 >>> """ def _get_forecast_scalar( system, Not_Used, rule_variation_name, this_stage, instrument_list, scalar_function, forecast_scalar_config): """ instrument_list contains multiple things, pools everything across all instruments """ print(__file__ + ":" + str(inspect.getframeinfo(inspect.currentframe())[:3][1]) + ":" +"Getting forecast scalar for %s over %s" % (rule_variation_name, ", ".join(instrument_list))) ## Get forecasts for each instrument forecast_list=[ this_stage.get_raw_forecast(instrument_code, rule_variation_name) for instrument_code in instrument_list] cs_forecasts=pd.concat(forecast_list, axis=1) scaling_factor=scalar_function(cs_forecasts, **forecast_scalar_config) return scaling_factor ## Get some useful stuff from the config forecast_scalar_config=copy(self.parent.config.forecast_scalar_estimate) # The config contains 'func' and some other arguments # we turn func which could be a string into a function, and then # call it with the other ags scalarfunction = resolve_function(forecast_scalar_config.pop('func')) ## this determines whether we pool or not pool_instruments=str2Bool(forecast_scalar_config.pop("pool_instruments")) if pool_instruments: ## pooled, same for all instruments instrument_code_key=ALL_KEYNAME instrument_list=self.parent.get_instrument_list() else: ## not pooled instrument_code_key=instrument_code instrument_list=[instrument_code] forecast_scalar = self.parent.calc_or_cache_nested( "get_forecast_scalar", instrument_code_key, rule_variation_name, _get_forecast_scalar, self, instrument_list, scalarfunction, forecast_scalar_config) return forecast_scalar
def with_class_object(self): class_of_entry_list = resolve_function(self.class_of_entry_list_as_str) return classWithListOfEntriesAsListOfDicts( class_of_entry_list, self.list_of_entries_as_list_of_dicts)
def calculation_of_pooled_raw_forecast_weights(self, instrument_code): """ Estimate the forecast weights for this instrument We store this intermediate step to expose the calculation object :param instrument_code: :type str: :returns: TxK pd.DataFrame containing weights, columns are trading rule variation names, T covers all """ def _calculation_of_pooled_raw_forecast_weights( system, instrument_code_ref, this_stage, codes_to_use, weighting_func, **weighting_params): this_stage.log.terse( "Calculating pooled raw forecast weights over instruments: %s" % instrument_code_ref) rule_list = self.apply_cost_weighting(instrument_code) weight_func = weighting_func(log=self.log.setup(call="weighting"), **weighting_params) if weight_func.need_data(): ## returns a list of accountCurveGroups ## cost pooling will already have been applied pandl_forecasts = [ this_stage.get_returns_for_optimisation(code) for code in codes_to_use ] ## have to decode these ## returns two lists of pd.DataFrames (pandl_forecasts_gross, pandl_forecasts_costs) = decompose_group_pandl( pandl_forecasts, pool_costs=True) ## The weighting function requires two lists of pd.DataFrames, one gross, one for costs weight_func.set_up_data(data_gross=pandl_forecasts_gross, data_costs=pandl_forecasts_costs) else: ## in the case of equal weights, don't need data forecasts = this_stage.get_all_forecasts( instrument_code, rule_list) weight_func.set_up_data(weight_matrix=forecasts) SR_cost_list = [ this_stage.get_SR_cost_for_instrument_forecast( instrument_code, rule_variation_name) for rule_variation_name in rule_list ] weight_func.optimise(ann_SR_costs=SR_cost_list) return weight_func ## Get some useful stuff from the config weighting_params = copy(self.parent.config.forecast_weight_estimate) ## do we pool our estimation? pooling_returns = str2Bool(weighting_params.pop("pool_gross_returns")) pooling_costs = self.parent.config.forecast_cost_estimates[ 'use_pooled_costs'] assert pooling_returns and pooling_costs ## which function to use for calculation weighting_func = resolve_function(weighting_params.pop("func")) codes_to_use = self.has_same_cheap_rules_as_code(instrument_code) instrument_code_ref = "_".join( codes_to_use) ## ensures we don't repeat optimisation ## ## _get_raw_forecast_weights: function to call if we don't find in cache ## self: this_system stage object ## codes_to_use: instrument codes to get data for ## weighting_func: function to call to calculate weights ## **weighting_params: parameters to pass to weighting function ## raw_forecast_weights_calcs = self.parent.calc_or_cache( 'calculation_of_raw_forecast_weights', instrument_code_ref, _calculation_of_pooled_raw_forecast_weights, self, codes_to_use, weighting_func, **weighting_params) return raw_forecast_weights_calcs
def get_forecast_correlation_matrices(self, instrument_code): """ Returns a correlationList object which contains a history of correlation matricies :param instrument_code: :type str: :returns: correlation_list object >>> from systems.tests.testdata import get_test_object_futures_with_rules_and_capping_estimate >>> from systems.basesystem import System >>> (accounts, fcs, rules, rawdata, data, config)=get_test_object_futures_with_rules_and_capping_estimate() >>> system=System([rawdata, rules, fcs, accounts, ForecastCombineEstimated()], data, config) >>> ans=system.combForecast.get_forecast_correlation_matrices("EDOLLAR") >>> ans.corr_list[-1] array([[ 1. , 0.1168699 , 0.08038547], [ 0.1168699 , 1. , 0.86907623], [ 0.08038547, 0.86907623, 1. ]]) >>> print(ans.columns) ['carry', 'ewmac16', 'ewmac8'] """ def _get_forecast_correlation_matrices(system, NotUsed1, NotUsed2, this_stage, codes_to_use, corr_func, **corr_params): print(__file__ + ":" + str(inspect.getframeinfo(inspect.currentframe())[:3][1]) + ":" +"Calculating forecast correlations over %s" % ", ".join(codes_to_use)) forecast_data=[this_stage.get_all_forecasts(instr_code, this_stage.apply_cost_weighting(instr_code)) for instr_code in codes_to_use] ## if we're not pooling passes a list of one forecast_data=[forecast_ts.ffill() for forecast_ts in forecast_data] return corr_func(forecast_data, log=self.log.setup(call="correlation"), **corr_params) ## Get some useful stuff from the config corr_params=copy(self.parent.config.forecast_correlation_estimate) ## do we pool our estimation? pooling=str2Bool(corr_params.pop("pool_instruments")) ## which function to use for calculation corr_func=resolve_function(corr_params.pop("func")) if pooling: ## find set of instruments with same trading rules as I have codes_to_use=self.has_same_cheap_rules_as_code(instrument_code) instrument_code_ref=ALL_KEYNAME ## We label='_'.join(codes_to_use) else: codes_to_use=[instrument_code] label=instrument_code instrument_code_ref=instrument_code ## ## label: how we identify this thing in the cache ## instrument_code_ref: eithier the instrument code, or 'all markets' if pooling ## _get_forecast_correlation_matrices: function to call if we don't find in cache ## self: this_system stage object ## codes_to_use: instrument codes ## func: function to call to calculate correlations ## **corr_params: parameters to pass to correlation function ## forecast_corr_list = self.parent.calc_or_cache_nested( 'get_forecast_correlation_matrices', instrument_code_ref, label, _get_forecast_correlation_matrices, self, codes_to_use, corr_func, **corr_params) return forecast_corr_list
def calculation_of_raw_forecast_weights_for_instrument(self, instrument_code): """ Does an optimisation for a single instrument We do this if we can't do the special case of a pooled optimisation Estimate the forecast weights for this instrument We store this intermediate step to expose the calculation object :param instrument_code: :type str: :returns: TxK pd.DataFrame containing weights, columns are trading rule variation names, T covers all """ def _calculation_of_raw_forecast_weights(system, instrument_code, this_stage, codes_to_use, weighting_func, pool_costs, **weighting_params): this_stage.log.terse("Calculating raw forecast weights for %s, over %s" % (instrument_code, ", ".join(codes_to_use))) rule_list = self.apply_cost_weighting(instrument_code) weight_func=weighting_func(log=self.log.setup(call="weighting"), **weighting_params) if weight_func.need_data(): ## returns a list of accountCurveGroups pandl_forecasts=[this_stage.get_returns_for_optimisation(code) for code in codes_to_use] ## the current curve is special pandl_forecasts_this_code=this_stage.get_returns_for_optimisation(instrument_code) ## have to decode these ## returns two lists of pd.DataFrames (pandl_forecasts_gross, pandl_forecasts_costs) = decompose_group_pandl(pandl_forecasts, pandl_forecasts_this_code, pool_costs=pool_costs) ## The weighting function requires two lists of pd.DataFrames, one gross, one for costs weight_func.set_up_data(data_gross = pandl_forecasts_gross, data_costs = pandl_forecasts_costs) else: ## in the case of equal weights, don't need data forecasts = this_stage.get_all_forecasts(instrument_code, rule_list) weight_func.set_up_data(weight_matrix=forecasts) SR_cost_list = [this_stage.get_SR_cost_for_instrument_forecast(instrument_code, rule_variation_name) for rule_variation_name in rule_list] weight_func.optimise(ann_SR_costs=SR_cost_list) return weight_func ## Get some useful stuff from the config weighting_params=copy(self.parent.config.forecast_weight_estimate) ## do we pool our estimation? pooling_returns = str2Bool(self.parent.config.forecast_weight_estimate["pool_gross_returns"]) pool_costs = str2Bool(self.parent.config.forecast_cost_estimates["use_pooled_costs"]) ## which function to use for calculation weighting_func=resolve_function(weighting_params.pop("func")) if pooling_returns: ## find set of instruments with same trading rules as I have codes_to_use=self.has_same_cheap_rules_as_code(instrument_code) else: codes_to_use=[instrument_code] ## ## _get_raw_forecast_weights: function to call if we don't find in cache ## self: this_system stage object ## codes_to_use: instrument codes to get data for ## weighting_func: function to call to calculate weights ## **weighting_params: parameters to pass to weighting function ## raw_forecast_weights_calcs = self.parent.calc_or_cache( 'calculation_of_raw_forecast_weights', instrument_code, _calculation_of_raw_forecast_weights, self, codes_to_use, weighting_func, pool_costs, **weighting_params) return raw_forecast_weights_calcs pass
def _calculation_of_raw_forecast_weights_for_instrument( self, instrument_code): """ Does an optimisation for a single instrument We do this if we can't do the special case of a fully pooled optimisation (both costs and returns pooled) Estimate the forecast weights for this instrument We store this intermediate step to expose the calculation object :param instrument_code: :type str: :returns: TxK pd.DataFrame containing weights, columns are trading rule variation names, T covers all """ # usual pool fix # Get some useful stuff from the config weighting_params = copy(self.parent.config.forecast_weight_estimate) # do we pool our estimation? (note if both are pooled would better calling the function for full pooling # eithier gross returns or costs can be pooled pooling_returns = str2Bool( self.parent.config.forecast_weight_estimate["pool_gross_returns"]) pool_costs = str2Bool( self.parent.config.forecast_cost_estimates["use_pooled_costs"]) # which function to use for calculation weighting_func = resolve_function(weighting_params.pop("func")) # FIXME: Returns and costs are pooled in different places, very confusing if pooling_returns: # find set of instruments with same trading rules as I have codes_to_use = self.has_same_cheap_rules_as_code(instrument_code) else: codes_to_use = [instrument_code] self.log.terse( "Calculating raw forecast weights for %s, over %s" % (instrument_code, ", ".join(codes_to_use))) rule_list = self.check_for_cheap_enough_rules(instrument_code) # FIXME: change the way log is passed to a 'parent' style weight_func = weighting_func( log=self.log.setup( call="weighting"), **weighting_params) # returns a list of accountCurveGroups pandl_forecasts = [self.get_returns_for_optimisation(code) for code in codes_to_use] # the current curve is special # FIXME couldn't the optimiser do this?: pandl_forecasts_this_code = self.get_returns_for_optimisation( instrument_code) # have to decode these # returns two lists of pd.DataFrames # FIXME: WHY do this? Instead get the optimiser to do it?? (pandl_forecasts_gross, pandl_forecasts_costs) = decompose_group_pandl( pandl_forecasts, pandl_forecasts_this_code, pool_costs=pool_costs) # The weighting function requires two lists of pd.DataFrames, # one gross, one for costs weight_func.set_up_data( data_gross=pandl_forecasts_gross, data_costs=pandl_forecasts_costs) weight_func.optimise() return weight_func
def get_forecast_scalar(self, instrument_code, rule_variation_name): """ Get the scalar to apply to raw forecasts If not cached, these are estimated from past forecasts If configuration variable pool_forecasts_for_scalar is "True", then we do this across instruments. :param instrument_code: :type str: :param rule_variation_name: :type str: name of the trading rule variation :returns: float >>> from systems.tests.testdata import get_test_object_futures_with_rules >>> from systems.basesystem import System >>> (rules, rawdata, data, config)=get_test_object_futures_with_rules() >>> system1=System([rawdata, rules, ForecastScaleCapEstimated()], data, config) >>> >>> ## From default >>> system1.forecastScaleCap.get_forecast_scalar("EDOLLAR", "ewmac8").tail(3) scale_factor 2015-12-09 5.849888 2015-12-10 5.850474 2015-12-11 5.851091 >>> system1.forecastScaleCap.get_capped_forecast("EDOLLAR", "ewmac8").tail(3) ewmac8 2015-12-09 0.645585 2015-12-10 -0.210377 2015-12-11 0.961821 >>> >>> ## From config >>> scale_config=dict(pool_instruments=False) >>> config.forecast_scalar_estimate=scale_config >>> system3=System([rawdata, rules, ForecastScaleCapEstimated()], data, config) >>> system3.forecastScaleCap.get_forecast_scalar("EDOLLAR", "ewmac8").tail(3) scale_factor 2015-12-09 5.652174 2015-12-10 5.652833 2015-12-11 5.653444 >>> """ def _get_forecast_scalar(system, Not_Used, rule_variation_name, this_stage, instrument_list, scalar_function, forecast_scalar_config): """ instrument_list contains multiple things, pools everything across all instruments """ this_stage.log.msg( "Getting forecast scalar for %s over %s" % (rule_variation_name, ", ".join(instrument_list)), rule_variation_name=rule_variation_name) # Get forecasts for each instrument forecast_list = [ this_stage.get_raw_forecast(instrument_code, rule_variation_name) for instrument_code in instrument_list ] cs_forecasts = pd.concat(forecast_list, axis=1) scaling_factor = scalar_function(cs_forecasts, **forecast_scalar_config) return scaling_factor # Get some useful stuff from the config forecast_scalar_config = copy( self.parent.config.forecast_scalar_estimate) # The config contains 'func' and some other arguments # we turn func which could be a string into a function, and then # call it with the other ags scalarfunction = resolve_function(forecast_scalar_config.pop('func')) # this determines whether we pool or not pool_instruments = str2Bool( forecast_scalar_config.pop("pool_instruments")) if pool_instruments: # pooled, same for all instruments instrument_code_key = ALL_KEYNAME instrument_list = self.parent.get_instrument_list() else: ## not pooled instrument_code_key = instrument_code instrument_list = [instrument_code] forecast_scalar = self.parent.calc_or_cache_nested( "get_forecast_scalar", instrument_code_key, rule_variation_name, _get_forecast_scalar, self, instrument_list, scalarfunction, forecast_scalar_config) return forecast_scalar
def calculation_of_raw_forecast_weights(self, instrument_code): """ Estimate the forecast weights for this instrument We store this intermediate step to expose the calculation object :param instrument_code: :type str: :returns: TxK pd.DataFrame containing weights, columns are trading rule variation names, T covers all """ def _calculation_of_raw_forecast_weights(system, instrument_code, this_stage, codes_to_use, weighting_func, pool_costs=False, **weighting_params): this_stage.log.terse("Calculating raw forecast weights over %s" % ", ".join(codes_to_use)) if hasattr(system, "accounts"): ## returns a list of accountCurveGroups pandl_forecasts=[this_stage.pandl_for_instrument_rules_unweighted(code) for code in codes_to_use] ## the current curve is special pandl_forecasts_this_code=this_stage.pandl_for_instrument_rules_unweighted(instrument_code) ## have to decode these ## returns two lists of pd.DataFrames (pandl_forecasts_gross, pandl_forecasts_costs) = decompose_group_pandl(pandl_forecasts, pandl_forecasts_this_code, pool_costs=pool_costs) else: error_msg="You need an accounts stage in the system to estimate forecast weights" this_stage.log.critical(error_msg) ## The weighting function requires two lists of pd.DataFrames, one gross, one for costs output=weighting_func(pandl_forecasts_gross, pandl_forecasts_costs, log=self.log.setup(call="weighting"), **weighting_params) return output ## Get some useful stuff from the config weighting_params=copy(self.parent.config.forecast_weight_estimate) ## do we pool our estimation? pooling=str2Bool(weighting_params.pop("pool_instruments")) ## which function to use for calculation weighting_func=resolve_function(weighting_params.pop("func")) if pooling: ## find set of instruments with same trading rules as I have codes_to_use=self._has_same_rules_as_code(instrument_code) else: codes_to_use=[instrument_code] ## ## _get_raw_forecast_weights: function to call if we don't find in cache ## self: this_system stage object ## codes_to_use: instrument codes to get data for ## weighting_func: function to call to calculate weights ## **weighting_params: parameters to pass to weighting function ## raw_forecast_weights_calcs = self.parent.calc_or_cache( 'calculation_of_raw_forecast_weights', instrument_code, _calculation_of_raw_forecast_weights, self, codes_to_use, weighting_func, **weighting_params) return raw_forecast_weights_calcs
def get_instrument_correlation_matrix(self): """ Returns a correlationList object which contains a history of correlation matricies :returns: correlation_list object >>> from systems.tests.testdata import get_test_object_futures_with_pos_sizing_estimates >>> from systems.basesystem import System >>> (account, posobject, combobject, capobject, rules, rawdata, data, config)=get_test_object_futures_with_pos_sizing_estimates() >>> system=System([rawdata, rules, posobject, combobject, capobject,PortfoliosEstimated(), account], data, config) >>> system.config.forecast_weight_estimate["method"]="shrinkage" ## speed things up >>> system.config.forecast_weight_estimate["date_method"]="in_sample" ## speed things up >>> system.config.instrument_weight_estimate["date_method"]="in_sample" ## speed things up >>> system.config.instrument_weight_estimate["method"]="shrinkage" ## speed things up >>> ans=system.portfolio.get_instrument_correlation_matrix() >>> ans.corr_list[-1] array([[ 1. , 0.56981346, 0.62458477], [ 0.56981346, 1. , 0.88087893], [ 0.62458477, 0.88087893, 1. ]]) >>> print(ans.corr_list[0]) [[ 1. 0.99 0.99] [ 0.99 1. 0.99] [ 0.99 0.99 1. ]] >>> print(ans.corr_list[10]) [[ 1. 0.99 0.99 ] [ 0.99 1. 0.78858156] [ 0.99 0.78858156 1. ]] """ def _get_instrument_correlation_matrix(system, NotUsed, this_stage, corr_func, **corr_params): this_stage.log.terse("Calculating instrument correlations") instrument_codes=system.get_instrument_list() if hasattr(system, "accounts"): pandl=this_stage.pandl_across_subsystems().to_frame() else: error_msg="You need an accounts stage in the system to estimate instrument correlations" this_stage.log.critical(error_msg) ## Need to resample here, because the correlation function won't do it properly frequency=corr_params['frequency'] pandl=pandl.cumsum().resample(frequency).diff() return corr_func(pandl, log=this_stage.log.setup(call="correlation"), **corr_params) ## Get some useful stuff from the config corr_params=copy(self.parent.config.instrument_correlation_estimate) ## which function to use for calculation corr_func=resolve_function(corr_params.pop("func")) ## _get_instrument_correlation_matrix: function to call if we don't find in cache ## self: this_system stage object ## func: function to call to calculate correlations ## **corr_params: parameters to pass to correlation function ## forecast_corr_list = self.parent.calc_or_cache( 'get_instrument_correlation_matrix', ALL_KEYNAME, _get_instrument_correlation_matrix, self, corr_func, **corr_params) return forecast_corr_list
def __init__(self, rule, data=list(), other_args=dict()): """ Create a trading rule from a function Functions must be of the form function(*dataargs, **kwargs), where *dataargs are unnamed data items, and **kwargs are named configuration items data, an ordered list of strings identifying data to be used (default, just price) other_args: a dictionary of named arguments to be passed to the trading rule :param rule: Trading rule to be created :type trading_rules: The following describe a rule completely (ignore data and other_args arguments) 3-tuple ; containing (function, data, other_args) dict (containing key "function", and optionally keys "other_args" and "data") TradingRule (object is created out of this rule) The following will be combined with the data and other_args arguments to produce a complete TradingRule: Other callable function str (with path to function eg "systems.provide.example.rules.ewmac_forecast_with_defaults") :param data: (list of) str pointing to location of inputs in a system method call (eg "data.get_instrument_price") (Eithier passed in separately, or as part of a TradingRule, 3-tuple, or dict object) :type data: single str, or list of str :param other_args: Other named arguments to be passed to trading rule function (Eithier passed in separately , or as part of a TradingRule, 3-tuple, or dict object) :type other_args: dict :returns: single Tradingrule object """ if hasallattr(rule, ["function", "data", "other_args"]): # looks like it is already a trading rule (rule_function, data, other_args) = ( rule.function, rule.data, rule.other_args) elif isinstance(rule, tuple): if len(data) > 0 or len(other_args) > 0: print( "WARNING: Creating trade rule with 'rule' tuple argument, ignoring data and/or other args") if len(rule) != 3: raise Exception( "Creating trading rule with a tuple, must be length 3 exactly (function/name, data [...], args dict(...))") (rule_function, data, other_args) = rule elif isinstance(rule, dict): if len(data) > 0 or len(other_args) > 0: print( "WARNING: Creating trade rule with 'rule' dict argument, ignoring data and/or other args") try: rule_function = rule['function'] except KeyError: raise Exception( "If you specify a TradingRule as a dict it has to contain a 'function' keyname") if "data" in rule: data = rule['data'] else: data = [] if "other_args" in rule: other_args = rule['other_args'] else: other_args = dict() else: rule_function = rule # turn string into a callable function if required rule_function = resolve_function(rule_function) if isinstance(data, str): # turn into a 1 item list or wont' get parsed properly data = [data] setattr(self, "function", rule_function) setattr(self, "data", data) setattr(self, "other_args", other_args)
def calculation_of_pooled_raw_forecast_weights(self, instrument_code): """ Estimate the forecast weights for this instrument We store this intermediate step to expose the calculation object :param instrument_code: :type str: :returns: TxK pd.DataFrame containing weights, columns are trading rule variation names, T covers all """ def _calculation_of_pooled_raw_forecast_weights(system, instrument_code_ref, this_stage, codes_to_use, weighting_func, **weighting_params): print(__file__ + ":" + str(inspect.getframeinfo(inspect.currentframe())[:3][1]) + ":" +"Calculating pooled raw forecast weights over instruments: %s" % instrument_code_ref) rule_list = self.apply_cost_weighting(instrument_code) weight_func=weighting_func(log=self.log.setup(call="weighting"), **weighting_params) if weight_func.need_data(): ## returns a list of accountCurveGroups ## cost pooling will already have been applied pandl_forecasts=[this_stage.get_returns_for_optimisation(code) for code in codes_to_use] ## have to decode these ## returns two lists of pd.DataFrames (pandl_forecasts_gross, pandl_forecasts_costs) = decompose_group_pandl(pandl_forecasts, pool_costs=True) ## The weighting function requires two lists of pd.DataFrames, one gross, one for costs weight_func.set_up_data(data_gross = pandl_forecasts_gross, data_costs = pandl_forecasts_costs) else: ## in the case of equal weights, don't need data forecasts = this_stage.get_all_forecasts(instrument_code, rule_list) weight_func.set_up_data(weight_matrix=forecasts) SR_cost_list = [this_stage.get_SR_cost_for_instrument_forecast(instrument_code, rule_variation_name) for rule_variation_name in rule_list] weight_func.optimise(ann_SR_costs=SR_cost_list) return weight_func ## Get some useful stuff from the config weighting_params=copy(self.parent.config.forecast_weight_estimate) ## do we pool our estimation? pooling_returns = str2Bool(weighting_params.pop("pool_gross_returns")) pooling_costs = self.parent.config.forecast_cost_estimates['use_pooled_costs'] assert pooling_returns and pooling_costs ## which function to use for calculation weighting_func=resolve_function(weighting_params.pop("func")) codes_to_use=self.has_same_cheap_rules_as_code(instrument_code) instrument_code_ref ="_".join(codes_to_use) ## ensures we don't repeat optimisation ## ## _get_raw_forecast_weights: function to call if we don't find in cache ## self: this_system stage object ## codes_to_use: instrument codes to get data for ## weighting_func: function to call to calculate weights ## **weighting_params: parameters to pass to weighting function ## raw_forecast_weights_calcs = self.parent.calc_or_cache( 'calculation_of_raw_forecast_weights', instrument_code_ref, _calculation_of_pooled_raw_forecast_weights, self, codes_to_use, weighting_func, **weighting_params) return raw_forecast_weights_calcs
def get_forecast_diversification_multiplier_estimated( self, instrument_code: str) -> pd.Series: """ Get the diversification multiplier for this instrument Estimated from correlations and weights :param instrument_code: instrument to get multiplier for :type instrument_code: str :returns: Tx1 pd.DataFrame >>> from systems.tests.testdata import get_test_object_futures_with_rules_and_capping_estimate >>> from systems.basesystem import System >>> (accounts, fcs, rules, rawdata, data, config)=get_test_object_futures_with_rules_and_capping_estimate() >>> system=System([accounts, rawdata, rules, fcs, ForecastCombineEstimated()], data, config) >>> system.config.forecast_weight_estimate['method']="shrinkage" >>> system.combForecast.get_forecast_diversification_multiplier("EDOLLAR").tail(3) FDM 2015-12-09 1.367351 2015-12-10 1.367349 2015-12-11 1.367347 >>> system.config.forecast_div_mult_estimate['dm_max']=1.1 >>> system=System([accounts, rawdata, rules, fcs, ForecastCombineEstimated()], data, system.config) >>> system.combForecast.get_forecast_diversification_multiplier("EDOLLAR").tail(3) FDM 2015-12-09 1.1 2015-12-10 1.1 2015-12-11 1.1 """ self.log.terse( "Calculating forecast div multiplier for %s" % instrument_code, instrument_code=instrument_code, ) # Get some useful stuff from the config div_mult_params = copy(self.parent.config.forecast_div_mult_estimate) # an example of an idm calculation function is # sysquant.estimators.diversification_multipliers.diversification_multiplier_from_list idm_func = resolve_function(div_mult_params.pop("func")) correlation_list = self.get_forecast_correlation_matrices( instrument_code) ## weights will be on same frequency as forecaster weight_df = self.get_forecast_weights(instrument_code) # note there is a possibility that the forecast_weights contain a subset of the rules in the correlation # matrices, because the forecast weights could have rules removed for being too expensive # To deal with this we pad the weights data frame so it is exactly # aligned with the correlations weight_df = dataframe_pad(weight_df, correlation_list.column_names, padwith=0.0) ts_fdm = idm_func(correlation_list, weight_df, **div_mult_params) return ts_fdm
def __init__(self, method, optimise_params, moments_estimator): print(__file__ + ":" + str(inspect.getframeinfo(inspect.currentframe())[:3][1]) + ":" + "optimiserWithParams") opt_func=bootstrap_portfolio setattr(self, "opt_func", resolve_function(opt_func)) setattr(self, "params", optimise_params) setattr(self, "moments_estimator", moments_estimator)
def _data_class(self): class_name = self._data_class_name() return resolve_function(class_name)
def get_empty_series_for_timed_entry(new_entry: timedEntry) -> listOfEntries: containing_data_class_name = new_entry.containing_data_class_name containing_data_class = resolve_function(containing_data_class_name) return containing_data_class.as_empty()
def create_broker_order_for_contract_order(self, contract_order_id, check_if_open=True): original_contract_order = self.contract_stack.get_order_with_id_from_stack(contract_order_id) log = original_contract_order.log_with_attributes(self.log) data_locks = dataLocks(self.data) instrument_locked = data_locks.is_instrument_locked(original_contract_order.instrument_code) if instrument_locked: log.msg("Instrument is locked, not spawning order") return None if check_if_open: data_broker = dataBroker(self.data) market_open = data_broker.is_instrument_code_and_contract_date_okay_to_trade(original_contract_order.instrument_code, original_contract_order.contract_id) if not market_open: return None # We can deal with partially filled contract orders: that's how hard we are! remaining_contract_order = original_contract_order.order_with_remaining() ## Check the order doesn't breach trade limits contract_order = self.what_contract_trade_is_possible(remaining_contract_order) ## Note we don't save the algo method, but reallocate each time ## This is useful if trading is about to finish, because we switch to market orders ## (assuming a bunch of limit orders haven't worked out so well) contract_order = check_and_if_required_allocate_algo_to_single_contract_order(self.data, contract_order) algo_to_use_str = contract_order.algo_to_use algo_method = resolve_function(algo_to_use_str) ## The algo method submits an order to the broker, and returns a broker order object ## We then save the brokerorder in the broker stack, and add it as a child to a contract order ## Algos may be 'fire and forget' (a simple market order, as implemented initially) or 'active' ## Active algos need to keep running on another thread (need to work out how to do this) ## They will set the property 'reference_of_controlling_algo' in contract order ## Fills are picked up by another process (or if the algo is an active thing, potentially by itself) broker_order, reference_of_controlling_algo = algo_method(self.data, contract_order) if broker_order is missing_order: # something bad has happened and we can't submit an order to the broker # Nae bother, maybe try again later # Unlock the contract order in case we want to do this later self.contract_stack.release_order_from_algo_control(contract_order_id) return None ## update trade limits self.add_trade_to_trade_limits(broker_order) broker_order_id = self.broker_stack.put_order_on_stack(broker_order) if type(broker_order_id) is not int: # We've created a broker order but can't add it to the broker order database # Probably safest to leave the contract order locked otherwise there could be multiple # broker orders issued and nobody wants that! log.critical("Created a broker order %s but can't add it to the order stack!! (condition %s)" % (str(broker_order), str(broker_order_id))) return failure # ....create new algo lock # This means nobody else can try and execute this order until it is released # Only the algo itself can release! # This only applies to 'fire and forget' orders that aren't controlled by an algo self.contract_stack.add_controlling_algo_ref(contract_order_id, reference_of_controlling_algo) # This broker order is a child of the parent contract order # We add 'another' child since it's valid to have multiple broker orders self.contract_stack.add_another_child_to_order(contract_order_id, broker_order_id) return success
def calculation_of_raw_forecast_weights(self, instrument_code): """ Estimate the forecast weights for this instrument We store this intermediate step to expose the calculation object :param instrument_code: :type str: :returns: TxK pd.DataFrame containing weights, columns are trading rule variation names, T covers all """ def _calculation_of_raw_forecast_weights(system, instrument_code, this_stage, codes_to_use, weighting_func, pool_costs=False, **weighting_params): this_stage.log.terse("Calculating raw forecast weights over %s" % ", ".join(codes_to_use)) if hasattr(system, "accounts"): ## returns a list of accountCurveGroups pandl_forecasts = [ this_stage.pandl_for_instrument_rules_unweighted(code) for code in codes_to_use ] ## the current curve is special pandl_forecasts_this_code = this_stage.pandl_for_instrument_rules_unweighted( instrument_code) ## have to decode these ## returns two lists of pd.DataFrames (pandl_forecasts_gross, pandl_forecasts_costs) = decompose_group_pandl( pandl_forecasts, pandl_forecasts_this_code, pool_costs=pool_costs) else: error_msg = "You need an accounts stage in the system to estimate forecast weights" this_stage.log.critical(error_msg) ## The weighting function requires two lists of pd.DataFrames, one gross, one for costs output = weighting_func(pandl_forecasts_gross, pandl_forecasts_costs, log=self.log.setup(call="weighting"), **weighting_params) return output ## Get some useful stuff from the config weighting_params = copy(self.parent.config.forecast_weight_estimate) ## do we pool our estimation? pooling = str2Bool(weighting_params.pop("pool_instruments")) ## which function to use for calculation weighting_func = resolve_function(weighting_params.pop("func")) if pooling: ## find set of instruments with same trading rules as I have codes_to_use = self._has_same_rules_as_code(instrument_code) else: codes_to_use = [instrument_code] ## ## _get_raw_forecast_weights: function to call if we don't find in cache ## self: this_system stage object ## codes_to_use: instrument codes to get data for ## weighting_func: function to call to calculate weights ## **weighting_params: parameters to pass to weighting function ## raw_forecast_weights_calcs = self.parent.calc_or_cache( 'calculation_of_raw_forecast_weights', instrument_code, _calculation_of_raw_forecast_weights, self, codes_to_use, weighting_func, **weighting_params) return raw_forecast_weights_calcs
def get_forecast_correlation_matrices(self, instrument_code): """ Returns a correlationList object which contains a history of correlation matricies :param instrument_code: :type str: :returns: correlation_list object >>> from systems.tests.testdata import get_test_object_futures_with_rules_and_capping_estimate >>> from systems.basesystem import System >>> (accounts, fcs, rules, rawdata, data, config)=get_test_object_futures_with_rules_and_capping_estimate() >>> system=System([rawdata, rules, fcs, accounts, ForecastCombineEstimated()], data, config) >>> ans=system.combForecast.get_forecast_correlation_matrices("EDOLLAR") >>> ans.corr_list[-1] array([[ 1. , 0.1168699 , 0.08038547], [ 0.1168699 , 1. , 0.86907623], [ 0.08038547, 0.86907623, 1. ]]) >>> print(ans.columns) ['carry', 'ewmac16', 'ewmac8'] """ def _get_forecast_correlation_matrices(system, NotUsed1, NotUsed2, this_stage, codes_to_use, corr_func, **corr_params): this_stage.log.terse("Calculating forecast correlations over %s" % ", ".join(codes_to_use)) forecast_data = [ this_stage.get_all_forecasts( instr_code, this_stage.apply_cost_weighting(instr_code)) for instr_code in codes_to_use ] ## if we're not pooling passes a list of one forecast_data = [ forecast_ts.ffill() for forecast_ts in forecast_data ] return corr_func(forecast_data, log=self.log.setup(call="correlation"), **corr_params) ## Get some useful stuff from the config corr_params = copy(self.parent.config.forecast_correlation_estimate) ## do we pool our estimation? pooling = str2Bool(corr_params.pop("pool_instruments")) ## which function to use for calculation corr_func = resolve_function(corr_params.pop("func")) if pooling: ## find set of instruments with same trading rules as I have codes_to_use = self.has_same_cheap_rules_as_code(instrument_code) instrument_code_ref = ALL_KEYNAME ## We label = '_'.join(codes_to_use) else: codes_to_use = [instrument_code] label = instrument_code instrument_code_ref = instrument_code ## ## label: how we identify this thing in the cache ## instrument_code_ref: eithier the instrument code, or 'all markets' if pooling ## _get_forecast_correlation_matrices: function to call if we don't find in cache ## self: this_system stage object ## codes_to_use: instrument codes ## func: function to call to calculate correlations ## **corr_params: parameters to pass to correlation function ## forecast_corr_list = self.parent.calc_or_cache_nested( 'get_forecast_correlation_matrices', instrument_code_ref, label, _get_forecast_correlation_matrices, self, codes_to_use, corr_func, **corr_params) return forecast_corr_list