def generate_features(self, new_df, input_dict):
funcDict = DataRetrieve.load_obj(self, 'func_dict')
for key in input_dict.keys():
print(key)
path = [key, 'fname']
print('fname: ', self.get_from_dict(input_dict, path))
func_name = self.get_from_dict(input_dict, path)
path = [key, 'params']
print('params: ', self.get_from_dict(input_dict, path))
params = self.get_from_dict(input_dict, path)
new_df = funcDict[func_name](new_df, *params)
#print(new_df.tail(10))
print("Current feature: ", current_feature['Latest'])
path = [key, 'transform']
print('transform: ', self.get_from_dict(input_dict, path))
do_transform = self.get_from_dict(input_dict, path)
if do_transform:
#print('!!!!', do_transform[0], )
pass_params = (do_transform[1::])
print("pass params", pass_params)
#print("Current feature: ", current_feature['Latest'])
new_df = funcDict[do_transform[0]](new_df,
current_feature['Latest'],
*pass_params)
#print("Current feature: ", current_feature['Latest'])
return new_df
"DetrendPO": oscSt.detrend_PO,
"ATR": vStud.ATR,
"NATR": vStud.NATR,
"ATRRatio": vStud.ATR_Ratio,
"DeltaATRRatio": vStud.delta_ATR_Ratio,
"BBWidth": vStud.BBWidth,
"HigherClose": candle_ind.higher_close,
"LowerClose": candle_ind.lower_close,
"ChaikinAD": taLibVolSt.ChaikinAD,
"ChaikinADOSC": taLibVolSt.ChaikinADOSC,
"OBV": taLibVolSt.OBV,
"MFI": taLibVolSt.MFI,
"ease_OfMvmnt": custVolSt.ease_OfMvmnt,
"exp_MA": taLibOS.exp_MA,
"simple_MA": taLibOS.simple_MA,
"weighted_MA": taLibOS.weighted_MA,
"triple_EMA": taLibOS.triple_EMA,
"triangMA": taLibOS.triangMA,
"dblEMA": taLibOS.dblEMA,
"kaufman_AMA": taLibOS.kaufman_AMA,
"delta_MESA_AMA": taLibOS.delta_MESA_AMA,
"inst_Trendline": taLibOS.inst_Trendline,
"mid_point": taLibOS.mid_point,
"mid_price": taLibOS.mid_price,
"pSAR": taLibOS.pSAR
}
dSet = DataRetrieve()
dSet.save_obj(functionDict, 'func_dict')
funcDict = dSet.load_obj('func_dict')
print(funcDict)
time_test_modelling.py
Goal: Test and verify in sample and out of sample time splits for dataset
"""
from Code.lib.plot_utils import PlotUtility
from Code.lib.time_utils import TimeUtility
from Code.lib.retrieve_data import DataRetrieve, ComputeTarget
from Code.utilities.stat_tests import stationarity_tests, mean_and_variance
import datetime
from dateutil.relativedelta import relativedelta
from pandas.tseries.offsets import BDay
if __name__ == "__main__":
plotIt = PlotUtility()
timeUtil = TimeUtility()
dSet = DataRetrieve()
ct = ComputeTarget()
doPlot = False
issue = "tlt"
pivotDate = datetime.date(2018, 4, 2)
is_oos_ratio = 2
oos_months = 3
segments = 3
# get segmented dates
isOosDates = timeUtil.is_oos_data_split(issue, pivotDate, is_oos_ratio,
oos_months, segments)
dataLoadStartDate = isOosDates[0]
is_start_date = isOosDates[1]
oos_start_date = isOosDates[2]
from Code.lib.retrieve_system_info import TradingSystemUtility
from Code.lib.candle_indicators import CandleIndicators
from Code.lib.transformers import Transformers
from Code.lib.ta_momentum_studies import TALibMomentumStudies
from Code.lib.model_utils import ModelUtility, TimeSeriesSplitImproved
from Code.lib.feature_generator import FeatureGenerator
from Code.lib.config import current_feature, feature_dict
from Code.models import models_utils
from Code.lib.model_algos import AlgoUtility
from Code.lib.tms_utils import TradeRisk
plotIt = PlotUtility()
timeUtil = TimeUtility()
ct = ComputeTarget()
candle_ind = CandleIndicators()
dSet = DataRetrieve()
taLibMomSt = TALibMomentumStudies()
transf = Transformers()
modelUtil = ModelUtility()
featureGen = FeatureGenerator()
dSet = DataRetrieve()
modelAlgo = AlgoUtility()
sysUtil = TradingSystemUtility()
tradeRisk = TradeRisk()
if __name__ == '__main__':
# set to existing system name OR set to blank if creating new
if len(sys.argv) < 2:
print('You must set a system_name or set to """"!!!')
"""
col_name = 'ROC_' + str(period)
current_feature['Latest'] = col_name
feature_dict[col_name] = 'Keep'
df[col_name] = ta.ROC(df.Close, period)
return df
if __name__ == "__main__":
from Code.lib.plot_utils import PlotUtility
from Code.lib.retrieve_data import DataRetrieve
plotIt = PlotUtility()
taLibMomSt = TALibMomentumStudies()
dSet = DataRetrieve()
issue = "TLT"
df = dSet.read_issue_data(issue)
lastRow = df.shape[0]
dataLoadEndDate = df.Date[lastRow - 100]
pprint(dataLoadEndDate)
dataLoadStartDate = df.Date[lastRow - 3000]
pprint(dataLoadStartDate)
dataSet = dSet.set_date_range(df, dataLoadStartDate, dataLoadEndDate)
pprint(dataSet.tail(10))
# Resolve any NA's for now
import pickle
#from pandas.tseries.holiday import USFederalHolidayCalendar
#from pandas.tseries.offsets import CustomBusinessDay
#us_cal = CustomBusinessDay(calendar=USFederalHolidayCalendar())
from sklearn.model_selection import StratifiedShuffleSplit
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier, GradientBoostingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn import svm
if __name__ == "__main__":
plotIt = PlotUtility()
timeUtil = TimeUtility()
ct = ComputeTarget()
candle_ind = CandleIndicators()
dSet = DataRetrieve()
taLibMomSt = TALibMomentumStudies()
transf = Transformers()
modelUtil = ModelUtility()
featureGen = FeatureGenerator()
issue = "TLT"
# Set IS-OOS parameters
pivotDate = datetime.date(2019, 1, 3)
is_oos_ratio = 2
oos_months = 4
segments = 4
df = dSet.read_issue_data(issue)
dataLoadStartDate = df.Date[0]
lastRow = df.shape[0]
#from Code.lib.model_utils import ModelUtility, TimeSeriesSplitImproved
#from Code.lib.feature_generator import FeatureGenerator
#from Code.utilities.stat_tests import stationarity_tests
#from Code.lib.config import current_feature, feature_dict
#from Code.models import models_utils
#from Code.lib.model_algos import AlgoUtility
from Code.lib.tms_utils import TradeRisk
plotIt = PlotUtility()
#timeUtil = TimeUtility()
#ct = ComputeTarget()
#taLibMomSt = TALibMomentumStudies()
#transf = Transformers()
#modelUtil = ModelUtility()
#featureGen = FeatureGenerator()
dSet = DataRetrieve()
#modelAlgo = AlgoUtility()
sysUtil = TradingSystemUtility()
tradeRisk = TradeRisk()
if __name__ == '__main__':
# set to existing system name OR set to blank if creating new
if len(sys.argv) < 2:
print('You must set a system_name or set to """"!!!')
system_name = sys.argv[1]
system_directory = sysUtil.get_system_dir(system_name)
#ext_input_dict = sys.argv[2]
print("Existing system")
for test_start in test_starts:
yield (indices[:test_start],
indices[test_start:test_start + test_size])
if __name__ == "__main__":
from Code.lib.plot_utils import PlotUtility
from Code.lib.retrieve_data import DataRetrieve, ComputeTarget
from Code.lib.time_utils import TimeUtility
from Code.lib.feature_generator import FeatureGenerator
from Code.utilities.stat_tests import stationarity_tests
# from Code.lib.config import current_feature, feature_dict
plotIt = PlotUtility()
ct = ComputeTarget()
dSet = DataRetrieve()
featureGen = FeatureGenerator()
timeUtil = TimeUtility()
modelUtil = ModelUtility()
#timeSeriesImp = TimeSeriesSplitImproved()
dataLoadStartDate = "2014-09-26"
dataLoadEndDate = "2018-04-05"
issue = "TLT"
# Set IS-OOS parameters
pivotDate = datetime.date(2018, 4, 2)
is_oos_ratio = 2
oos_months = 4
segments = 4
dataSet = dSet.read_issue_data(issue)
# -*- coding: utf-8 -*-
"""
Created on Wed Mar 6 14:30:17 2019
@author: KRUEGKJ
"""
import os
import datetime
import random
from Code.lib.retrieve_data import DataRetrieve
dSet = DataRetrieve()
current_directory = os.getcwd()
# create
list_of_input_dicts = []
def gen_dict_name():
# generate random input_dict name
a = random.sample(range(10),3)
sys_date = datetime.datetime.now().strftime("%Y%m%d")
name = "input_dict_" + str(sys_date) + "_" + str(a[0]) + str(a[1]) + str(a[2])
return name
test_input_dict = {} # initialize
test_input_dict = {'f1':
{'fname' : 'PPO',
'params' : [2,5],
'transform' : ['Normalized', 20]
},
# -*- coding: utf-8 -*-
"""
Created on Mon Mar 4 11:06:54 2019
@author: KRUEGKJ
"""
import random
import os
import os.path
import sys
from Code.lib.retrieve_data import DataRetrieve
dSet = DataRetrieve()
class TradingSystemUtility:
# Function to generate system number
def get_sys_number(self):
# Generate four digit random number
a = random.sample(range(10), 3)
number = int(
str(random.randint(1, 9)) + str(a[0]) + str(a[1]) + str(a[2]))
return number
# function to create system directory based on system name if one doesn't already exist
def get_system_dir(self, system_name):
# Create system directory in current path
current_directory = os.getcwd()
system_directory = os.path.join(current_directory, system_name)
if not os.path.exists(system_directory):
os.makedirs(system_directory)
from Code.lib.retrieve_system_info import TradingSystemUtility
#from Code.lib.candle_indicators import CandleIndicators
from Code.lib.transformers import Transformers
#from Code.lib.ta_momentum_studies import TALibMomentumStudies
from Code.lib.model_utils import ModelUtility, TimeSeriesSplitImproved
from Code.lib.feature_generator import FeatureGenerator
#from Code.utilities.stat_tests import stationarity_tests
from Code.lib.config import current_feature, feature_dict
from Code.models import models_utils
from Code.lib.model_algos import AlgoUtility
plotIt = PlotUtility()
timeUtil = TimeUtility()
ct = ComputeTarget()
#candle_ind = CandleIndicators()
dSet = DataRetrieve()
#taLibMomSt = TALibMomentumStudies()
transf = Transformers()
modelUtil = ModelUtility()
featureGen = FeatureGenerator()
modelAlgo = AlgoUtility()
sysUtil = TradingSystemUtility()
if __name__ == '__main__':
# set to existing system name OR set to blank if creating new
if len(sys.argv) < 2:
print('You must set a system_name or set to """"!!!')
system_name = sys.argv[1]
ext_input_dict = sys.argv[2]
from Code.lib.candle_indicators import CandleIndicators
from Code.lib.transformers import Transformers
from Code.lib.ta_momentum_studies import TALibMomentumStudies
from Code.lib.model_utils import ModelUtility, TimeSeriesSplitImproved
from Code.lib.feature_generator import FeatureGenerator
from Code.utilities.stat_tests import stationarity_tests
from Code.lib.config import current_feature, feature_dict
from Code.models import models_utils
from Code.lib.model_algos import AlgoUtility
from Code.lib.tms_utils import TradeRisk
plotIt = PlotUtility()
timeUtil = TimeUtility()
ct = ComputeTarget()
candle_ind = CandleIndicators()
dSet = DataRetrieve()
taLibMomSt = TALibMomentumStudies()
transf = Transformers()
modelUtil = ModelUtility()
featureGen = FeatureGenerator()
dSet = DataRetrieve()
modelAlgo = AlgoUtility()
sysUtil = TradingSystemUtility()
tradeRisk = TradeRisk()
if __name__ == '__main__':
from Code.lib.plot_utils import PlotUtility
plotIt = PlotUtility()
# set to existing system name OR set to blank if creating new
if len(sys.argv) < 2:
"""
Created on Fri Mar 8 17:04:07 2019
@author: kruegkj
Re-use existing input_dicts to generate new results
"""
import os
import glob
from Code.lib.retrieve_data import DataRetrieve
current_directory = os.getcwd()
dSet = DataRetrieve()
# create
list_of_input_dicts = []
num_dicts_to_make = 40
dict_ctr = 0
dicts_to_use = glob.glob("input_dict*")
program_name = "IS_Day_by_day_TLT_v1.py"
for dict_name in dicts_to_use:
print(dict_name)
current_directory = os.getcwd()
# add try/catch block later
from Code.lib.retrieve_system_info import TradingSystemUtility
from Code.lib.candle_indicators import CandleIndicators
from Code.lib.transformers import Transformers
from Code.lib.ta_momentum_studies import TALibMomentumStudies
from Code.lib.model_utils import ModelUtility, TimeSeriesSplitImproved
from Code.lib.feature_generator import FeatureGenerator
from Code.utilities.stat_tests import stationarity_tests
from Code.lib.config import current_feature, feature_dict
from Code.models import models_utils
from Code.lib.model_algos import AlgoUtility
plotIt = PlotUtility()
timeUtil = TimeUtility()
ct = ComputeTarget()
candle_ind = CandleIndicators()
dSet = DataRetrieve()
taLibMomSt = TALibMomentumStudies()
transf = Transformers()
modelUtil = ModelUtility()
featureGen = FeatureGenerator()
dSet = DataRetrieve()
modelAlgo = AlgoUtility()
sysUtil = TradingSystemUtility()
if __name__ == '__main__':
# set to existing system name OR set to blank if creating new
if len(sys.argv) < 2:
print('You must set a system_name or set to """"!!!')
system_name = sys.argv[1]