import sys; sys.path.append('../..') import logging;logging.basicConfig(stream=sys.stdout,level=logging.DEBUG,format='%(asctime)s - %(message)s - %(pathname)s') from systems.provided.example.rules import ewmac_forecast_with_defaults as ewmac from sysdata.configdata import Config from systems.account import Account from systems.forecast_combine import ForecastCombine from systems.forecast_scale_cap import ForecastScaleCap from systems.basesystem import System from sysdata.csvdata import csvFuturesData from systems.forecasting import Rules from systems.forecasting import TradingRule data = csvFuturesData() my_config = Config() ewmac_8 = TradingRule((ewmac, [], dict(Lfast=8, Lslow=32))) ewmac_32 = TradingRule(dict(function=ewmac, other_args=dict(Lfast=32, Lslow=128))) my_rules = Rules(dict(ewmac8=ewmac_8, ewmac32=ewmac_32)) my_config.trading_rules = dict(ewmac8=ewmac_8, ewmac32=ewmac_32) #my_config.instruments=[ "US20", "NASDAQ", "SP500"] my_config.instruments=[ "SP500"] my_config.forecast_weight_estimate=dict(method="bootstrap") my_config.forecast_weight_estimate['monte_runs']=50 my_config.use_forecast_weight_estimates=True my_system = System([Account(), ForecastScaleCap(), my_rules, ForecastCombine()], data, my_config) logging.debug(my_system.combForecast.get_forecast_weights("SP500").tail(5)) # DEBUG:root: ewmac32 ewmac8
def __init__(self, stage_list, data, config=None, log=logtoscreen("base_system")): """ Create a system object for doing simulations or live trading :param stage_list: A list of stages :type stage_list: list of systems.stage.SystemStage (or anything that inherits from it) :param data: data for doing simulations :type data: sysdata.data.simData (or anything that inherits from that) :param config: Optional configuration :type config: sysdata.configdata.Config :returns: new system object >>> from systems.stage import SystemStage >>> stage=SystemStage() >>> from sysdata.sim.csv_futures_sim_data import csvFuturesSimData >>> data=csvFuturesSimData() >>> System([stage], data) System base_system with .config, .data, and .stages: unnamed """ if config is None: # Default - for very dull systems this is sufficient config = Config() setattr(self, "data", data) setattr(self, "config", config) self._log = log self.config._system_init(self) self.data._system_init(self) stage_names = [] try: iter(stage_list) except AssertionError: raise Exception( "You didn't pass a list into this System instance; even just one stage should be System([stage_instance])" ) for stage in stage_list: """ This is where we put the methods to store various stages of the process """ # Stages have names, which are also how we find them in the system # attributes sub_name = stage.name # Each stage has a link back to the parent system # This init sets this, and also passes the system logging object stage._system_init(self) if sub_name in stage_names: raise Exception( "You have duplicate subsystems with the name %s. Remove " "one of them, or change a name." % sub_name) setattr(self, sub_name, stage) stage_names.append(sub_name) setattr(self, "_stage_names", stage_names) """ The cache hides all intermediate results We call optimal_positions and then that propogates back finding all the data we need The results are then cached in the object. Should we call delete_instrument_data (in base class system) then everything related to a particular instrument is removed from these 'nodes' except for protected items This is very useful in live trading when we don't want to update eg cross sectional data every sample """ setattr(self, "cache", systemCache(self)) self.name = "base_system" # makes caching work and for general consistency
subsys_positions = pd.concat(subsys_positions, axis=1).ffill() subsys_positions.columns = instrument_list instrument_weights = fix_weights_vs_pdm(raw_instr_weights, subsys_positions) weighting=system.config.instrument_weight_ewma_span instrument_weights = pd.ewma(instrument_weights, weighting) return instrument_weights if __name__ == "__main__": random.seed(0) np.random.seed(0) data = csvFuturesData() my_config = Config() my_config.instruments=["US20", "SP500"] ewmac_8 = TradingRule((ewmac, [], dict(Lfast=8, Lslow=32))) ewmac_32 = TradingRule(dict(function=ewmac, other_args=dict(Lfast=32, Lslow=128))) my_rules = Rules(dict(ewmac8=ewmac_8, ewmac32=ewmac_32)) my_system = System([Account(), PortfoliosEstimated(), PositionSizing(), ForecastScaleCap(), my_rules, ForecastCombine()], data, my_config) my_system.config.forecast_weight_estimate['method']="equal_weights" my_system.config.instrument_weight_estimate['method']="bootstrap" my_system.config.instrument_weight_estimate["monte_runs"]=1 my_system.config.instrument_weight_estimate["bootstrap_length"]=250 print(my_system.portfolio.get_instrument_diversification_multiplier(my_system)) # 10,250 weights=0.75,0.25 idm=1.26 # 30,250 weights=0.75,0.25
from systems.provided.example.simplesystem import simplesystem my_system = simplesystem(log_level="on") print(my_system) print(my_system.portfolio.get_notional_position("EDOLLAR").tail(5)) from sysdata.csv.csv_sim_futures_data import csvFuturesSimData from sysdata.configdata import Config """ Now loading config and data """ my_config = Config("systems.provided.example.simplesystemconfig.yaml") my_data = csvFuturesSimData() my_system = simplesystem(config=my_config, data=my_data) print(my_system.portfolio.get_notional_position("EDOLLAR").tail(5)) """ Let's get the chapter 15 system """ from systems.provided.futures_chapter15.basesystem import futures_system from matplotlib.pyplot import show system = futures_system(log_level="on") print(system.accounts.portfolio().sharpe()) system.accounts.portfolio().curve().plot() show() """ Same for estimated system """
from sysdata.configdata import Config from systems.account import Account from systems.forecast_combine import ForecastCombine from systems.forecast_scale_cap import ForecastScaleCap from systems.basesystem import System from sysdata.csvdata import csvFuturesData from systems.forecasting import Rules from systems.forecasting import TradingRule data = csvFuturesData() ewmac_8 = TradingRule((ewmac, [], dict(Lfast=8, Lslow=32))) ewmac_32 = TradingRule(dict(function=ewmac, other_args=dict(Lfast=32, Lslow=128))) my_rules = Rules(dict(ewmac8=ewmac_8, ewmac32=ewmac_32)) my_config = Config() my_config my_config.trading_rules = dict(ewmac8=ewmac_8, ewmac32=ewmac_32) ## we can estimate these ourselves #my_config.instruments=[ "US20", "NASDAQ", "SP500"] my_config.instruments=[ "SP500"] #my_config.forecast_weight_estimate=dict(method="one_period") my_config.forecast_weight_estimate=dict(method="bootstrap") my_config.forecast_weight_estimate['monte_runs']=50 my_config.use_forecast_weight_estimates=True my_account = Account() combiner = ForecastCombine() fcs=ForecastScaleCap() my_system = System([my_account, fcs, my_rules, combiner], data, my_config)
from sysdata.configdata import Config from systems.account import Account from systems.forecast_combine import ForecastCombine from systems.forecast_scale_cap import ForecastScaleCap from systems.basesystem import System from sysdata.csvdata import csvFuturesData from systems.forecasting import Rules from systems.forecasting import TradingRule data = csvFuturesData() ewmac_8 = TradingRule((ewmac, [], dict(Lfast=8, Lslow=32))) ewmac_32 = TradingRule(dict(function=ewmac, other_args=dict(Lfast=32, Lslow=128))) my_rules = Rules(dict(ewmac8=ewmac_8, ewmac32=ewmac_32)) my_config = Config() my_config my_config.trading_rules = dict(ewmac8=ewmac_8, ewmac32=ewmac_32) ## we can estimate these ourselves #my_config.instruments=[ "US20", "NASDAQ", "SP500"] my_config.instruments=[ "SP500"] my_config.forecast_weight_estimate=dict(method="one_period") my_config.use_forecast_weight_estimates=True my_account = Account() combiner = ForecastCombine() fcs=ForecastScaleCap() my_system = System([my_account, fcs, my_rules, combiner], data, my_config) print(my_system.combForecast.get_forecast_weights("SP500").tail(5))
""" ... or two... """ ewmac_8 = TradingRule((ewmac, [], dict(Lfast=8, Lslow=32))) ewmac_32 = TradingRule( dict(function=ewmac, other_args=dict(Lfast=32, Lslow=128))) my_rules = Rules(dict(ewmac8=ewmac_8, ewmac32=ewmac_32)) print(my_rules.trading_rules()['ewmac32']) my_system = System([my_rules], data) my_system.rules.get_raw_forecast("EDOLLAR", "ewmac32").tail(5) from sysdata.configdata import Config my_config = Config() my_config empty_rules = Rules() my_config.trading_rules = dict(ewmac8=ewmac_8, ewmac32=ewmac_32) my_system = System([empty_rules], data, my_config) my_system.rules.get_raw_forecast("EDOLLAR", "ewmac32").tail(5) from systems.forecast_scale_cap import ForecastScaleCapFixed my_config.forecast_scalars = dict(ewmac8=5.3, ewmac32=2.65) fcs = ForecastScaleCapFixed() my_system = System([fcs, my_rules], data, my_config) print(my_system.forecastScaleCap.get_capped_forecast( "EDOLLAR", "ewmac32").tail(5))
return new_position simple_mav_rule = TradingRule( dict(function=simple_mav, other_args=dict(long=2, short=8))) simple_mav_rule_with_vol = TradingRule( dict(function=mav_with_vol, data=["rawdata.get_daily_prices", "rawdata.daily_returns_volatility"], other_args=dict(long=16, short=4))) """ now run the basic system; 16,64 crossover plus X=8 """ # weights = dict(EDOLLAR=1.0) config = Config( dict(trading_rules=dict(simple_mav=simple_mav_rule), Xfactor=8, percentage_vol_target=16.0)) config_2 = Config( dict(trading_rules=dict(simple_mav_with_vol=simple_mav_rule_with_vol), Xfactor=8, percentage_vol_target=16.0)) config_3 = Config( dict(trading_rules=dict(simple_mav_with_vol=simple_mav_rule_with_vol), Xfactor=8, percentage_vol_target=16.0, use_forecast_scale_estimates=True)) data = csvFuturesSimData()
from sysdata.configdata import Config from systems.account import Account from systems.forecast_combine import ForecastCombine from systems.forecast_scale_cap import ForecastScaleCap from systems.basesystem import System from sysdata.csvdata import csvFuturesData from systems.forecasting import Rules from systems.forecasting import TradingRule data = csvFuturesData() ewmac_8 = TradingRule((ewmac, [], dict(Lfast=8, Lslow=32))) ewmac_32 = TradingRule(dict(function=ewmac, other_args=dict(Lfast=32, Lslow=128))) my_rules = Rules(dict(ewmac8=ewmac_8, ewmac32=ewmac_32)) my_config = Config() my_config my_config.trading_rules = dict(ewmac8=ewmac_8, ewmac32=ewmac_32) ## we can estimate these ourselves #my_config.instruments=[ "US20", "NASDAQ", "SP500"] my_config.instruments=[ "SP500"] my_config.forecast_weight_estimate=dict(method="one_period") my_config.use_forecast_weight_estimates=True my_account = Account() combiner = ForecastCombine() fcs=ForecastScaleCap() my_system = System([my_account, fcs, my_rules, combiner], data, my_config) logging.debug ('get_forecast_weights')
return my_rules def random_system_for_regression(config, rules, log_level="on"): my_system = System([Account(), PortfoliosFixed(), PositionSizing(), ForecastCombineEstimated(), ForecastScaleCapEstimated(), rules, RawData()], csvFuturesData(), config) my_system.set_logging_level(log_level) return my_system rules=create_rules_for_random_system() config=Config(dict(use_forecast_scale_estimates = True, use_forecast_weight_estimates=True, forecast_scalar_estimate=dict(pool_instruments=True), instrument_weights=dict(EDOLLAR=.25, US10=.25, CORN=.25, SP500=.25), instrument_div_multiplier=1.5)) system=random_system_for_regression(config, rules) system.accounts.portfolio().cumsum().plot() show() ans=system.accounts.pandl_for_all_trading_rules_unweighted() ans.gross.to_frame().cumsum().plot() legend() show()
allrulespandl=system.accounts.pandl_for_all_trading_rules() ## ewmac_all=allrulespandl.to_frame().loc[:,evariations].sum(axis=1) break_all=allrulespandl.to_frame().loc[:,bvariations].sum(axis=1) both_plot=pd.concat([ewmac_all, break_all], axis=1) print(both_plot.corr()) both_plot.plot() show() """ ## full backtest compare my_config = Config("examples.breakout.breakoutfuturesestimateconfig.yaml") ## will do all instruments we have data for del (my_config.instruments) ## temporarily remove breakout rules my_config.rule_variations = evariations my_config.forecast_weight_estimate["method"] = "equal_weights" system_old = futures_system(config=my_config, log_level="on") ## new system has all trading rules new_config = Config("examples.breakout.breakoutfuturesestimateconfig.yaml") new_config.rule_variations = bvariations new_config.forecast_weight_estimate["method"] = "equal_weights" del (new_config.instruments) system_new = futures_system(config=new_config, log_level="on")
from systems.provided.futures_chapter15.estimatedsystem import futures_system from sysdata.configdata import Config my_config = Config("examples.breakout.breakoutfuturesestimateconfig.yaml") system = futures_system(config=my_config) system.config.notional_trading_capital = 100000 system.set_logging_level("on") system.accounts.portfolio().stats() system.portfolio.get_instrument_weights().plot() from matplotlib.pyplot import show show()
def __init__(self, stage_list, data, config=None, log=logtoscreen("base_system")): """ Create a system object for doing simulations or live trading :param stage_list: A list of stages :type stage_list: list of systems.stage.SystemStage (or anything that inherits from it) :param data: data for doing simulations :type data: sysdata.data.Data (or anything that inherits from that) :param config: Optional configuration :type config: sysdata.configdata.Config :returns: new system object >>> from systems.stage import SystemStage >>> stage=SystemStage() >>> from sysdata.csvdata import csvFuturesData >>> data=csvFuturesData() >>> System([stage], data) System with stages: unnamed """ if config is None: # Default - for very dull systems this is sufficient config = Config() config.fill_with_defaults() setattr(self, "data", data) setattr(self, "config", config) setattr(self, "log", log) setattr(data, "log", log.setup(stage="data")) setattr(config, "log", log.setup(stage="config")) protected = [] nopickle = [] stage_names = [] assert isinstance(stage_list, list) for stage in stage_list: """ This is where we put the methods to store various stages of the process """ # Stages have names, which are also how we find them in the system # attributes sub_name = stage.name # Each stage has a link back to the parent system stage._system_init(self) # and a log log = log.setup(stage=sub_name) setattr(stage, "log", log) if sub_name in stage_names: raise Exception( "You have duplicate subsystems with the name %s. Remove " "one of them, or change a name." % sub_name) setattr(self, sub_name, stage) stage_names.append(sub_name) # list of attributes / methods of the stage which are protected stage_protected = getattr(stage, "_protected", []) stage_protected = [(sub_name, protected_item, "*") for protected_item in stage_protected] protected += stage_protected stage_nopickle = getattr(stage, "_nopickle", []) stage_nopickle = [(sub_name, protected_item, "*") for protected_item in stage_nopickle] nopickle += stage_nopickle setattr(self, "_stage_names", stage_names) """ The cache hides all intermediate results We call optimal_positions and then that propogates back finding all the data we need The results are then cached in the object. Should we call delete_instrument_data (in base class system) then everything related to a particular instrument is removed from these 'nodes' except for protected items This is very useful in live trading when we don't want to update eg cross sectional data every sample """ setattr(self, "_cache", dict()) setattr(self, "_protected", protected) setattr(self, "_nopickle", nopickle)
from systems.provided.futures_chapter15.basesystem import futures_system from systems.provided.example.rules import ewmac_forecast_with_defaults as ewmac from systems.forecasting import TradingRule from systems.forecasting import Rules from sysdata.configdata import Config my_config = Config("interface.futuresconfig.yaml") # Config object system = futures_system(log_level="on", config=my_config) # We parse the config object acording the class, for basesystem, the keyword is config, this change # accordding to class, as you can see we don't need to parse the stages.It makes everything # futures_system does but with customs atributes. portfolio = system.accounts.portfolio() print(system.positionSize.get_subsystem_position("EDOLLAR").tail(2)) # The output is somenthing like below # ss_position # 2015-12-10 1.811465 # 2015-12-11 2.544598
def __init__(self, stage_list, data, config=None, log=logtoscreen("base_system")): """ Create a system object for doing simulations or live trading :param stage_list: A list of stages :type stage_list: list of systems.stage.SystemStage (or anything that inherits from it) :param data: data for doing simulations :type data: sysdata.data.Data (or anything that inherits from that) :param config: Optional configuration :type config: sysdata.configdata.Config :returns: new system object >>> from systems.stage import SystemStage >>> stage=SystemStage() >>> from sysdata.csvdata import csvFuturesData >>> data=csvFuturesData() >>> System([stage], data) System with stages: unnamed """ if config is None: # Default - for very dull systems this is sufficient config = Config() config.fill_with_defaults() setattr(self, "data", data) setattr(self, "config", config) setattr(self, "log", log) setattr(data, "log", log.setup(stage="data")) setattr(config, "log", log.setup(stage="config")) protected = [] nopickle=[] stage_names = [] assert isinstance(stage_list, list) for stage in stage_list: """ This is where we put the methods to store various stages of the process """ # Stages have names, which are also how we find them in the system # attributes sub_name = stage.name # Each stage has a link back to the parent system stage._system_init(self) # and a log log=log.setup(stage=sub_name) setattr(stage, "log", log) if sub_name in stage_names: raise Exception( "You have duplicate subsystems with the name %s. Remove " "one of them, or change a name." % sub_name) setattr(self, sub_name, stage) stage_names.append(sub_name) # list of attributes / methods of the stage which are protected stage_protected = getattr(stage, "_protected", []) stage_protected = [(sub_name, protected_item, "*") for protected_item in stage_protected] protected += stage_protected stage_nopickle=getattr(stage, "_nopickle", []) stage_nopickle = [(sub_name, protected_item, "*") for protected_item in stage_nopickle] nopickle += stage_nopickle setattr(self, "_stage_names", stage_names) """ The cache hides all intermediate results We call optimal_positions and then that propogates back finding all the data we need The results are then cached in the object. Should we call delete_instrument_data (in base class system) then everything related to a particular instrument is removed from these 'nodes' except for protected items This is very useful in live trading when we don't want to update eg cross sectional data every sample """ setattr(self, "_cache", dict()) setattr(self, "_protected", protected) setattr(self, "_nopickle", nopickle)
ewmac_rule """ ... or two... """ ewmac_8 = TradingRule((ewmac, [], dict(Lfast=8, Lslow=32))) ewmac_32 = TradingRule( dict(function=ewmac, other_args=dict(Lfast=32, Lslow=128))) my_rules = Rules(dict(ewmac8=ewmac_8, ewmac32=ewmac_32)) print(my_rules.trading_rules()['ewmac32']) my_system = System([my_rules], data) my_system.rules.get_raw_forecast("EDOLLAR", "ewmac32").tail(5) from sysdata.configdata import Config my_config = Config() my_config empty_rules = Rules() my_config.trading_rules = dict(ewmac8=ewmac_8, ewmac32=ewmac_32) my_system = System([empty_rules], data, my_config) my_system.rules.get_raw_forecast("EDOLLAR", "ewmac32").tail(5) from systems.forecast_scale_cap import ForecastScaleCap # we can estimate these ourselves my_config.instruments = ["US10", "EDOLLAR", "CORN", "SP500"] my_config.use_forecast_scale_estimates = True fcs = ForecastScaleCap() my_system = System([fcs, my_rules], data, my_config) print(
allrulespandl=system.accounts.pandl_for_all_trading_rules() ## ewmac_all=allrulespandl.to_frame().loc[:,evariations].sum(axis=1) break_all=allrulespandl.to_frame().loc[:,bvariations].sum(axis=1) both_plot=pd.concat([ewmac_all, break_all], axis=1) print(both_plot.corr()) both_plot.plot() show() """ # full backtest compare my_config = Config("examples.breakout.breakoutfuturesestimateconfig.yaml") # will do all instruments we have data for del (my_config.instruments) # temporarily remove breakout rules my_config.rule_variations = evariations my_config.forecast_weight_estimate["method"] = "equal_weights" system_old = futures_system(config=my_config, log_level="on") # new system has all trading rules new_config = Config("examples.breakout.breakoutfuturesestimateconfig.yaml") new_config.rule_variations = bvariations new_config.forecast_weight_estimate["method"] = "equal_weights" del (new_config.instruments) system_new = futures_system(config=new_config, log_level="on")
subsys_positions = pd.concat(subsys_positions, axis=1).ffill() subsys_positions.columns = instrument_list instrument_weights = fix_weights_vs_pdm(raw_instr_weights, subsys_positions) weighting=system.config.instrument_weight_ewma_span instrument_weights = pd.ewma(instrument_weights, weighting) return instrument_weights if __name__ == "__main__": random.seed(0) np.random.seed(0) data = csvFuturesData() my_config = Config() my_config.instruments=["US20", "SP500"] ewmac_8 = TradingRule((ewmac, [], dict(Lfast=8, Lslow=32))) ewmac_32 = TradingRule(dict(function=ewmac, other_args=dict(Lfast=32, Lslow=128))) my_rules = Rules(dict(ewmac8=ewmac_8, ewmac32=ewmac_32)) my_system = System([Account(), PortfoliosEstimated(), PositionSizing(), ForecastScaleCap(), my_rules, ForecastCombine()], data, my_config) my_system.config.forecast_weight_estimate['method']="equal_weights" my_system.config.instrument_weight_estimate['method']="bootstrap" my_system.config.instrument_weight_estimate["monte_runs"]=1 my_system.config.instrument_weight_estimate["bootstrap_length"]=250 #print(my_system.portfolio.get_instrument_diversification_multiplier(my_system)) print(my_system.portfolio.get_instrument_correlation_matrix(my_system).corr_list) # 10,250 weights=0.75,0.25 idm=1.26
""" ... or two... """ ewmac_8 = TradingRule((ewmac, [], dict(Lfast=8, Lslow=32))) ewmac_32 = TradingRule( dict(function=ewmac, other_args=dict(Lfast=32, Lslow=128))) my_rules = Rules(dict(ewmac8=ewmac_8, ewmac32=ewmac_32)) print(my_rules.trading_rules()['ewmac32']) my_system = System([my_rules], data) my_system.rules.get_raw_forecast("EDOLLAR", "ewmac32").tail(5) from sysdata.configdata import Config my_config = Config() my_config empty_rules = Rules() my_config.trading_rules = dict(ewmac8=ewmac_8, ewmac32=ewmac_32) my_system = System([empty_rules], data, my_config) my_system.rules.get_raw_forecast("EDOLLAR", "ewmac32").tail(5) from systems.forecast_scale_cap import ForecastScaleCap ## we can estimate these ourselves my_config.instruments=[ "US10", "EDOLLAR", "CORN", "SP500"] my_config.use_forecast_scale_estimates=True fcs=ForecastScaleCap()
import pandas as pd from pprint import pprint from matplotlib import pyplot as plt from private.estimatedsystemavanti import futures_system from sysdata.configdata import Config from sysdata.csvdata import csvFuturesData pandl_df = pd.DataFrame() # Initializing system. my_config = Config("private.nocarryconfig2.yaml") data = csvFuturesData("private.noCarryData") #This is the path to the CSV data. system = futures_system(log_level="on", config=my_config, data=data) # Making the backtest for the complete portfolio. instruments = system.config.instruments portfolio = system.accounts.portfolio() info = {} for item in instruments: series = [] series.append(data.daily_prices(item)) series.append(system.accounts.get_buffered_position(item)) series.append(system.combForecast.get_combined_forecast(item)) series.append(portfolio.to_frame()[item]) series.append(portfolio.to_frame()[item].cumsum())