def production_classic_futures_system(
data,
config_filename,
log=logtoscreen("futures_system"),
notional_trading_capital=None,
base_currency=None,
):
log_level = "on"
sim_data = dataSimData(data)
config = Config(config_filename)
# Overwrite capital
if notional_trading_capital is not None:
config.notional_trading_capital = notional_trading_capital
if base_currency is not None:
config.base_currency = base_currency
system = futures_system(data=sim_data, config=config)
system._log = log
system.set_logging_level(log_level)
return system
def test_simple_system_config_import(self, data):
my_config = Config("systems.provided.example.simplesystemconfig.yaml")
my_config.risk_overlay = arg_not_supplied
my_config.exclude_instrument_lists = dict(
ignore_instruments=["MILK"],
trading_restrictions=["BUTTER"],
bad_markets=["CHEESE"],
)
print(my_config)
my_system = System(
[
Account(),
Portfolios(),
PositionSizing(),
ForecastCombine(),
ForecastScaleCap(),
Rules(),
RawData(),
],
data,
my_config,
)
print(my_system.rules.get_raw_forecast("EDOLLAR", "ewmac32").tail(5))
print(my_system.rules.get_raw_forecast("EDOLLAR", "ewmac8").tail(5))
print(
my_system.forecastScaleCap.get_capped_forecast("EDOLLAR", "ewmac32").tail(5)
)
print(my_system.forecastScaleCap.get_forecast_scalar("EDOLLAR", "ewmac32"))
print(my_system.combForecast.get_combined_forecast("EDOLLAR").tail(5))
print(my_system.combForecast.get_forecast_weights("EDOLLAR").tail(5))
print(my_system.positionSize.get_subsystem_position("EDOLLAR").tail(5))
print(my_system.portfolio.get_notional_position("EDOLLAR").tail(5))
def production_classic_futures_system(
data: dataBlob,
config_filename: str,
log=logtoscreen("futures_system"),
notional_trading_capital: float = arg_not_supplied,
base_currency: str = arg_not_supplied,
) -> System:
log_level = "on"
sim_data = get_sim_data_object_for_production(data)
config = Config(config_filename)
# Overwrite capital and base currency
if notional_trading_capital is not arg_not_supplied:
config.notional_trading_capital = notional_trading_capital
if base_currency is not arg_not_supplied:
config.base_currency = base_currency
system = futures_system(data=sim_data, config=config)
system._log = log
system.set_logging_level(log_level)
return system
def testRules(self):
# config=Config(dict(trading_rules=dict(ewmac=dict(function="systems.provided.rules.ewmac.ewmac_forecast_with_defaults"))))
NOTUSEDrawdata, data, NOTUSEDconfig = get_test_object()
rules = Rules(
dict(function="systems.provided.rules.ewmac.ewmac_forecast_with_defaults")
)
system = System([rules], data)
ans = system.rules.get_raw_forecast("EDOLLAR", "rule0")
self.assertAlmostEqual(ans.tail(1).values[0], -3.280028, 5)
config = Config(
dict(
trading_rules=dict(
ewmac=dict(
function="systems.provided.rules.ewmac.ewmac_forecast_with_defaults"
)
)
)
)
rules = Rules()
system = System([rules], data, config)
ans = system.rules.get_raw_forecast("EDOLLAR", "ewmac")
self.assertAlmostEqual(ans.tail(1).values[0], -3.28002839, 5)
config = Config("systems.provided.example.exampleconfig.yaml")
rawdata = RawData()
rules = Rules()
system = System([rules, rawdata], data, config)
ans = system.rules.get_raw_forecast("EDOLLAR", "ewmac8")
self.assertAlmostEqual(ans.tail(1).values[0], -2.158634, 5)
def test_prebaked_from_confg(self):
"""
This is the config system from 'examples.introduction.prebakedsimplesystems'
"""
my_config = Config("systems.provided.example.simplesystemconfig.yaml")
my_config.risk_overlay = arg_not_supplied
my_data = csvFuturesSimData()
my_system = simplesystem(config=my_config, data=my_data)
print(my_system.portfolio.get_notional_position("EDOLLAR").tail(5))
def get_control_config() -> Config:
try:
control_config = Config(PRIVATE_CONTROL_CONFIG_FILE,
default_filename=DEFAULT_CONTROL_CONFIG_FILE)
control_config.fill_with_defaults()
except:
raise Exception("Need to have either %s or %s or both present:" %
(str(DEFAULT_CONTROL_CONFIG_FILE),
str(PRIVATE_CONTROL_CONFIG_FILE)))
return control_config
def set_up_config(data: dataBlob, config_filename: str) -> Config:
production_config = data.config
backtest_file_config = Config(config_filename)
# 'later elements overwrite earlier ones'
config = Config([production_config, backtest_file_config])
## this is also done by the system, but more transparent to do it here
config.fill_with_defaults()
return config
def simplesystem(data=None, config=None, log_level="on"):
"""
Example of how to 'wrap' a complete system
"""
if config is None:
config = Config("systems.provided.example.simplesystemconfig.yaml")
if data is None:
data = csvFuturesSimData()
my_system = System(
[
Account(),
Portfolios(),
PositionSizing(),
ForecastCombine(),
ForecastScaleCap(),
Rules(),
],
data,
config,
)
my_system.set_logging_level(log_level)
return my_system
def futures_system(sim_data=arg_not_supplied,
config_filename="systems.provided.rob_system.config.yaml"):
if sim_data is arg_not_supplied:
sim_data = dbFuturesSimData()
config = Config(config_filename)
system = System(
[
Risk(),
accountForOptimisedStage(),
optimisedPositions(),
Portfolios(),
PositionSizing(),
myFuturesRawData(),
ForecastCombine(),
volAttenForecastScaleCap(),
Rules(),
],
sim_data,
config,
)
system.set_logging_level("on")
return system
def test_mp_optimisation(self):
config = Config()
weighting_params = config.default_config_dict[
'forecast_weight_estimate']
n_threads = 8
weighting_params[
'n_threads'] = n_threads # running from a Pool of threads
start_time1 = time()
weights1 = optimiseWeightsOverTime(self.net_returns,
**weighting_params).weights()
stop_time1 = time()
del weighting_params[
'n_threads'] # running in main process, not separate process
start_time2 = time()
weights2 = optimiseWeightsOverTime(self.net_returns,
**weighting_params).weights()
stop_time2 = time()
print(
f"Takes {stop_time1-start_time1} seconds with {n_threads} processes"
)
print(
f"Takes {stop_time2-start_time2} seconds running in main process")
self.assertTrue(weights1.equals(weights2))
def generate_matching_duplicate_dict(config: Config):
"""
Returns a dict, each element is a named set of duplicated instruments
Within each dict we have two elements: included, excluded
Each of these is a list
For example:
dict(copper = dict(included = ["COPPER"], excluded = ["COPPER_mini"]
"""
duplicate_instruments_config = config.get_element_or_missing_data(
"duplicate_instruments")
if duplicate_instruments_config is missing_data:
raise Exception("Need 'duplicate_instruments' in config")
exclude_dict = duplicate_instruments_config.get("exclude", missing_data)
include_dict = duplicate_instruments_config.get("include", missing_data)
if exclude_dict is missing_data or include_dict is missing_data:
raise Exception(
"Need 'duplicate_instruments': exclude_dict and include_dict in config"
)
joint_keys = list(
set(list(exclude_dict.keys()) + list(include_dict.keys())))
results_dict = dict([(key,
get_duplicate_dict_entry(key, include_dict,
exclude_dict))
for key in joint_keys])
return results_dict
def get_config_of_excluded_instruments(config: Config) -> dict:
exclude_instrument_lists = config.get_element_or_missing_data(
"exclude_instrument_lists")
if exclude_instrument_lists is missing_data:
return {}
return exclude_instrument_lists
def get_control_config() -> Config:
try:
control_config = Config(
PRIVATE_CONTROL_CONFIG_FILE, default_filename=DEFAULT_CONTROL_CONFIG_FILE
)
control_config.fill_with_defaults()
except ParserError as pe:
raise Exception("YAML syntax problem: %s" % str(pe))
except FileNotFoundError:
raise Exception(
"Need to have either %s or %s or both present:"
% (str(DEFAULT_CONTROL_CONFIG_FILE), str(PRIVATE_CONTROL_CONFIG_FILE))
)
except BaseException as be:
raise Exception("Problem reading control config: %s" % str(be))
return control_config
def get_test_object_futures():
"""
Returns some standard test data
"""
data = csvFuturesSimData()
rawdata = FuturesRawData()
config = Config("systems.provided.example.exampleconfig.yaml")
return (rawdata, data, config)
def setUp(self):
system = System(
[testStage1(), testStage2()],
simData(),
Config(dict(instruments=["code", "another_code"])),
)
self.system = system
def get_test_object_futures_with_rules_and_capping():
"""
Returns some standard test data
"""
data = csvFuturesSimData()
rawdata = FuturesRawData()
rules = Rules()
config = Config("systems.provided.example.exampleconfig.yaml")
capobject = ForecastScaleCap()
return (capobject, rules, rawdata, data, config)
def setUp(self):
class testStage(SystemStage):
@property
def name(self):
return "test"
stage = testStage()
data = simData()
config = Config(dict(instruments=["another_code", "code"]))
system = System([stage], data=data, config=config)
self.system = system
def get_test_object_futures_with_pos_sizing():
"""
Returns some standard test data
"""
data = csvFuturesSimData()
rawdata = FuturesRawData()
rules = Rules()
config = Config("systems.provided.example.exampleconfig.yaml")
capobject = ForecastScaleCap()
combobject = ForecastCombine()
posobject = PositionSizing()
return (posobject, combobject, capobject, rules, rawdata, data, config)
def get_test_object_futures():
"""
Returns some standard test data
"""
data = csvFuturesSimData(datapath_dict=dict(
config_data="sysdata.tests.configtestdata",
adjusted_prices="sysdata.tests.adjtestdata",
spot_fx_data="sysdata.tests.fxtestdata",
multiple_price_data="sysdata.tests.multiplepricestestdata",
))
rawdata = FuturesRawData()
config = Config("systems.provided.example.exampleconfig.yaml")
return (rawdata, data, config)
def generate_matching_duplicate_dict(config: Config = arg_not_supplied):
"""
Returns a dict, each element is a named set of duplicated instruments
Within each dict we have two elements: included, excluded
Each of these is a list
For example:
dict(copper = dict(included = ["COPPER"], excluded = ["COPPER_mini"]
"""
if config is arg_not_supplied:
print(
"Using defaults.yaml config - won't include any elements from private_config or backtest configs"
)
config = Config()
config.fill_with_defaults()
duplicate_instruments_config = config.get_element_or_missing_data(
"duplicate_instruments")
if duplicate_instruments_config is missing_data:
raise Exception("Need 'duplicate_instruments' in config")
exclude_dict = duplicate_instruments_config.get("exclude", missing_data)
include_dict = duplicate_instruments_config.get("include", missing_data)
if exclude_dict is missing_data or include_dict is missing_data:
raise Exception(
"Need 'duplicate_instruments': exclude_dict and include_dict in config"
)
joint_keys = list(
set(list(exclude_dict.keys()) + list(include_dict.keys())))
results_dict = dict([(key,
get_duplicate_dict_entry(key, include_dict,
exclude_dict))
for key in joint_keys])
return results_dict
def system():
"""test fixture creates a system with start and end dates"""
system = futures_system(
data=CsvFuturesSimTestData(),
config=Config("systems.provided.futures_chapter15.futuresconfig.yaml"),
)
# speed things up
system.config.forecast_weight_estimate["method"] = "shrinkage"
system.config.forecast_weight_estimate["date_method"] = "in_sample"
system.config.instrument_weight_estimate["date_method"] = "in_sample"
system.config.instrument_weight_estimate["method"] = "shrinkage"
return system
def get_test_object_futures_with_rules_and_capping():
"""
Returns some standard test data
"""
data = csvFuturesSimData(datapath_dict=dict(
config_data="sysdata.tests.configtestdata",
adjusted_prices="sysdata.tests.adjtestdata",
spot_fx_data="sysdata.tests.fxtestdata",
multiple_price_data="sysdata.tests.multiplepricestestdata",
))
rawdata = FuturesRawData()
rules = Rules()
config = Config("systems.provided.example.exampleconfig.yaml")
capobject = ForecastScaleCap()
return (capobject, rules, rawdata, data, config)
def get_production_config() -> Config:
if private_config_file_exists():
config = Config(PRIVATE_CONFIG_FILE)
else:
print("Private configuration %s does not exist; no problem if running in sim mode")
config = Config({})
config.fill_with_defaults()
return config
def futures_system(data=None,
config=None,
trading_rules=arg_not_supplied,
log_level="on"):
"""
:param data: data object (defaults to reading from csv files)
:type data: sysdata.data.simData, or anything that inherits from it
:param config: Configuration object (defaults to futuresconfig.yaml in this directory)
:type config: sysdata.configdata.Config
:param trading_rules: Set of trading rules to use (defaults to set specified in config object)
:param trading_rules: list or dict of TradingRules, or something that can be parsed to that
:param log_level: Set of trading rules to use (defaults to set specified in config object)
:type log_level: str
"""
if data is None:
data = csvFuturesSimData()
if config is None:
config = Config(
"systems.provided.futures_chapter15.futuresestimateconfig.yaml")
rules = Rules(trading_rules)
system = System(
[
Account(),
Portfolios(),
PositionSizing(),
FuturesRawData(),
ForecastCombine(),
ForecastScaleCap(),
rules,
],
data,
config,
)
system.set_logging_level(log_level)
return system
def my_config(ewmac_8, ewmac_32):
my_config = Config()
my_config.trading_rules = dict(ewmac8=ewmac_8, ewmac32=ewmac_32)
my_config.instruments = ["US10", "EDOLLAR", "CORN", "SP500"]
my_config.risk_overlay = arg_not_supplied
my_config.exclude_instrument_lists = dict(
ignore_instruments=["MILK"],
trading_restrictions=["BUTTER"],
bad_markets=["CHEESE"],
)
return my_config
def test_simple_system_trading_rules_fixed(self, data, my_rules, fcs):
# or we can use the values from the book
my_config = Config()
my_config.trading_rules = dict(ewmac8=ewmac_8, ewmac32=ewmac_32)
my_config.instruments = ["US10", "EDOLLAR", "CORN", "SP500"]
my_config.forecast_scalars = dict(ewmac8=5.3, ewmac32=2.65)
my_config.use_forecast_scale_estimates = False
my_system = System([fcs, my_rules], data, my_config)
print(
my_system.forecastScaleCap.get_capped_forecast("EDOLLAR", "ewmac32").tail(5)
)
def get_test_object_futures_with_pos_sizing_estimates():
"""
Returns some standard test data
"""
data = csvFuturesSimData(datapath_dict=dict(
config_data="sysdata.tests.configtestdata",
adjusted_prices="sysdata.tests.adjtestdata",
spot_fx_data="sysdata.tests.fxtestdata",
multiple_price_data="sysdata.tests.multiplepricestestdata",
))
rawdata = FuturesRawData()
rules = Rules()
config = Config("systems.provided.example.estimateexampleconfig.yaml")
capobject = ForecastScaleCap()
combobject = ForecastCombine()
posobject = PositionSizing()
account = Account()
return (account, posobject, combobject, capobject, rules, rawdata, data,
config)
def __init__(
self,
stage_list: list,
data: simData,
config: Config = arg_not_supplied,
log: logger = logtoscreen("base_system"),
):
"""
Create a system object for doing simulations or live trading
:param stage_list: A list of stages
:type stage_list: list of systems.stage.SystemStage (or anything that inherits from it)
:param data: data for doing simulations
:type data: sysdata.data.simData (or anything that inherits from that)
:param config: Optional configuration
:type config: sysdata.configdata.Config
:returns: new system object
>>> from systems.stage import SystemStage
>>> stage=SystemStage()
>>> from sysdata.sim.csv_futures_sim_data import csvFuturesSimData
>>> data=csvFuturesSimData()
>>> System([stage], data)
System base_system with .config, .data, and .stages: Need to replace method when inheriting
"""
if config is arg_not_supplied:
# Default - for very dull systems this is sufficient
config = Config()
self._data = data
self._config = config
self._log = log
self.config.system_init(self)
self.data.system_init(self)
self._setup_stages(stage_list)
self._cache = systemCache(self)
def test_simple_system_risk_overlay(self, data, ewmac_8, ewmac_32):
my_config = Config(
dict(
trading_rules=dict(ewmac8=ewmac_8, ewmac32=ewmac_32),
instrument_weights=dict(US10=0.1, EDOLLAR=0.4, CORN=0.3, SP500=0.2),
instrument_div_multiplier=1.5,
forecast_scalars=dict(ewmac8=5.3, ewmac32=2.65),
forecast_weights=dict(ewmac8=0.5, ewmac32=0.5),
forecast_div_multiplier=1.1,
percentage_vol_target=25.00,
notional_trading_capital=500000,
base_currency="GBP",
risk_overlay=dict(
max_risk_fraction_normal_risk=1.4,
max_risk_fraction_stdev_risk=3.6,
max_risk_limit_sum_abs_risk=3.4,
max_risk_leverage=13.0,
),
exclude_instrument_lists=dict(
ignore_instruments=["MILK"],
trading_restrictions=["BUTTER"],
bad_markets=["CHEESE"],
),
)
)
print(my_config)
my_system = System(
[
Account(),
Portfolios(),
PositionSizing(),
ForecastCombine(),
ForecastScaleCap(),
Rules(),
RawData(),
],
data,
my_config,
)
print(my_system.portfolio.get_notional_position("EDOLLAR").tail(5))
def get_duplicate_list_of_instruments_to_remove_from_config(
config: Config) -> list:
duplicate_instruments_config = config.get_element_or_missing_data(
"duplicate_instruments")
if duplicate_instruments_config is missing_data:
return []
exclude_dict = duplicate_instruments_config.get("exclude", missing_data)
if exclude_dict is missing_data:
return []
list_of_duplicates = list(exclude_dict.values())
## do this because can have multiple duplicates
duplicate_list_flattened = []
for item in list_of_duplicates:
if type(item) is list:
duplicate_list_flattened = duplicate_list_flattened + item
else:
duplicate_list_flattened.append(item)
return duplicate_list_flattened