def SampleCNN(ticker: str): #Print sample CNN graphs of ticker, CNN will treat price data as picture and anticipate the next picture plot = PlotHelper() prices = PricingData(ticker) print('Loading ' + ticker) if prices.LoadHistory(): prices.NormalizePrices() window_size = 80 target_size = 10 daysInTraining = 800 sampleData = prices.GetPriceHistory() endDate = sampleData.index.max() cuttoffDate = endDate - BDay(window_size) startDate = cuttoffDate - BDay(daysInTraining) print(dataFolder + 'samples\CNNsampleLearning', startDate, cuttoffDate, endDate) for i in range(0, 10): ii = i * window_size d1 = startDate + BDay(ii) d2 = d1 + BDay(target_size) print(d1, d2, window_size, target_size) plot.PlotDataFrameDateRange( sampleData[['Average']], d1, window_size, 'Sample image ' + str(i), 'Date', 'Price', dataFolder + 'samples/CNN' + str(i) + 'Sample') plot.PlotDataFrameDateRange( sampleData[['Average']], d2, target_size, 'Target image ' + str(i), 'Date', 'Price', dataFolder + 'samples/CNN' + str(i) + 'Target')
def CalculatePriceCorrelation(tickerList: list): datafileName = 'data/_priceCorrelations.csv' summaryfileName = 'data/_priceCorrelationTop10.txt' result = pandas.DataFrame() startDate = str(datetime.datetime.now().date() + datetime.timedelta(days=-365)) endDate = str(datetime.datetime.now().date()) for ticker in tickerList: prices = PricingData(ticker) print('Loading ' + ticker) if prices.LoadHistory(True): prices.TrimToDateRange(startDate, endDate) prices.NormalizePrices() result[ticker] = prices.GetPriceHistory(['Average']) result = result.corr() result.to_csv(datafileName) f = open(summaryfileName, 'w') for ticker in tickerList: topTen = result.nsmallest(10, ticker) print(topTen[ticker]) f.write(ticker + '\n') f.write(topTen[ticker].to_string(header=True, index=True) + '\n') f.write('\n') f.close()
def CalculatePriceCorrelation(tickerList: list): datafileName = 'data/_priceCorrelations.csv' summaryfileName = 'data/_priceCorrelationTop10.txt' result = None startDate = str(datetime.datetime.now().date() + datetime.timedelta(days=-365)) endDate = str(datetime.datetime.now().date()) for ticker in tickerList: prices = PricingData(ticker) print('Loading ' + ticker) if prices.LoadHistory(): prices.TrimToDateRange(startDate, endDate) prices.NormalizePrices() x = prices.GetPriceHistory(['Average']) x.rename(index=str, columns={"Average": ticker}, inplace=True) if result is None: result = x else: result = result.join(x, how='outer') result = result.corr() result.to_csv(datafileName) f = open(summaryfileName, 'w') for ticker in tickerList: topTen = result.nsmallest(10, ticker) print(topTen[ticker]) f.write(ticker + '\n') f.write(topTen[ticker].to_string(header=True, index=True) + '\n') f.write('\n') f.close()
def CalculatePriceCorrelation(tickerList: list): datafileName = 'data/_priceCorrelations.csv' summaryfileName = 'data/_priceCorrelationTop10.txt' result = None startDate = str(AddDays(GetTodaysDate(), -365)) endDate = str(GetTodaysDate()) for ticker in tickerList: prices = PricingData(ticker) print('Loading ' + ticker) if prices.LoadHistory(requestedEndDate=GetTodaysDate()): prices.TrimToDateRange(startDate, endDate) prices.NormalizePrices() x = prices.GetPriceHistory(['Average']) x.rename(index=str, columns={"Average": ticker}, inplace=True) if result is None: result = x else: result = result.join(x, how='outer') result = result.corr() result.to_csv(datafileName) f = open(summaryfileName, 'w') for ticker in tickerList: topTen = result.nsmallest(10, ticker) print(topTen[ticker]) f.write(ticker + '\n') f.write(topTen[ticker].to_string(header=True, index=True) + '\n') f.write('\n') f.close() print( 'Intended to create stability, in practice, this is a great way to pair well performing stocks with poor performing or volatile stocks.' )
def SampleGraphs(ticker:str, daysInGraph:int): plot = PlotHelper() prices = PricingData(ticker) print('Loading ' + ticker) if prices.LoadHistory(True): prices.NormalizePrices() sampleData = prices.GetPriceHistory() d = sampleData.index[-1] for i in range(0,200, 10): #Add new days to the end for crystal ball predictions sampleDate = d - BDay(i) #pick business day to plot plot.PlotDataFrameDateRange(sampleData[['Open','High', 'Low','Close']], sampleDate, daysInGraph, 'Sample window ' + str(daysInGraph), 'Date', 'Price', dataFolder + 'samples/sample' + str(i) + '_' + str(daysInGraph))
def PredictPrices(prices:PricingData, predictionMethod:int=0, daysForward:int = 4, numberOfLearningPasses:int = 500): #Simple procedure to test different prediction methods assert(0 <= predictionMethod <= 2) plot = PlotHelper() if predictionMethod ==0: #Linear projection print('Running Linear Projection model predicting ' + str(daysForward) + ' days...') modelDescription = prices.stockTicker + '_Linear_daysforward' + str(daysForward) predDF = prices.GetPriceHistory() predDF['Average'] = (predDF['Open'] + predDF['High'] + predDF['Low'] + predDF['Close'])/4 d = predDF.index[-1] for i in range(0,daysForward): #Add new days to the end for crystal ball predictions predDF.loc[d + BDay(i+1), 'Average_Predicted'] = 0 predDF['PastSlope'] = predDF['Average'].shift(daysForward) / predDF['Average'].shift(daysForward*2) predDF['Average_Predicted'] = predDF['Average'].shift(daysForward) * predDF['PastSlope'] predDF['PercentageDeviation'] = abs((predDF['Average']-predDF['Average_Predicted'])/predDF['Average']) else: learningModule = StockPredictionNN() SourceFieldList = ['High','Low','Open','Close'] if predictionMethod ==1: #LSTM learning print('Running LSTM model predicting ' + str(daysForward) + ' days...') window_size = 1 modelDescription = prices.stockTicker + '_LSTM' + '_epochs' + str(numberOfLearningPasses) + '_histwin' + str(window_size) + '_daysforward' + str(daysForward) learningModule.LoadData(prices.GetPriceHistory(), window_size=window_size, prediction_target_days=daysForward, UseLSTM=True, SourceFieldList=SourceFieldList, batch_size=10, train_test_split=.93) learningModule.TrainLSTM(epochs=numberOfLearningPasses, learning_rate=2e-5, dropout_rate=0.8, gradient_clip_margin=4) elif predictionMethod ==2: #CNN Learning print('Running CNN model predicting ' + str(daysForward) + ' days...') window_size = 16 * daysForward modelDescription = prices.stockTicker + '_CNN' + '_epochs' + str(numberOfLearningPasses) + '_histwin' + str(window_size) + '_daysforward' + str(daysForward) learningModule.LoadData(prices.GetPriceHistory(), window_size=window_size, prediction_target_days=daysForward, UseLSTM=False, SourceFieldList=SourceFieldList, batch_size=32, train_test_split=.93) learningModule.TrainCNN(epochs=numberOfLearningPasses) predDF = learningModule.GetTrainingResults(True, True) averageDeviation = predDF['PercentageDeviation'].tail(round(predDF.shape[0]/4)).mean() #Average of the last 25% to account for training. print('Average deviation: ', averageDeviation * 100, '%') predDF = predDF.reindex(sorted(predDF.columns), axis=1) #Sort columns alphabetical predDF.to_csv(dataFolder + modelDescription + '.csv') plot.PlotDataFrame(predDF[['Average','Average_Predicted']], modelDescription, 'Date', 'Price', True, 'experiment/' + modelDescription) plot.PlotDataFrameDateRange(predDF[['Average','Average_Predicted']], None, 160, modelDescription + '_last160ays', 'Date', 'Price', dataFolder + modelDescription + '_last160Days') plot.PlotDataFrameDateRange(predDF[['Average','Average_Predicted']], None, 1000, modelDescription + '_last1000ays', 'Date', 'Price', dataFolder + modelDescription + '_last1000Days')
def SampleLSTM(ticker:str): plot = PlotHelper() prices = PricingData(ticker) print('Loading ' + ticker) CreateFolder(dataFolder + 'samples') if prices.LoadHistory(True): prices.NormalizePrices() daysInTarget = 15 daysInTraining = 200 sampleData = prices.GetPriceHistory() endDate = sampleData.index.max() cuttoffDate = endDate - BDay(daysInTarget) startDate = cuttoffDate - BDay(daysInTraining) print(dataFolder + 'samples/LSTMsampleLearning', startDate, cuttoffDate, endDate) plot.PlotDataFrameDateRange(sampleData[['Average']], cuttoffDate, daysInTraining, 'Learn from this series of days', 'Date', 'Price', dataFolder + 'samples/LSTMLearning') plot.PlotDataFrameDateRange(sampleData[['Average']], endDate, daysInTarget, 'Predict what happens after this series of days', 'Date', 'Price', dataFolder + 'samples/LSTMTarget')
def PlotAnnualPerformance(ticker: str = '^SPX'): print('Annual performance rate for ' + ticker) prices = PricingData(ticker) if prices.LoadHistory(True): x = prices.GetPriceHistory(['Average']) yearly = x.groupby([(x.index.year)]).first() yearlyChange = yearly.pct_change(1) monthly = x.groupby([(x.index.year), (x.index.month)]).first() plot = PlotHelper() plot.PlotDataFrame(yearly, title='Yearly', adjustScale=False) plot.PlotDataFrame(monthly, title='Monthly', adjustScale=False) plot.PlotDataFrame(yearlyChange, title='Yearly Percentage Change', adjustScale=False) print('Average annual change from ', prices.historyStartDate, ' to ', prices.historyEndDate, ': ', yearlyChange.mean().values * 100, '%')
def TestPredictionModels(ticker: str = '^SPX', numberOfLearningPasses: int = 300): #Simple procedure to test different prediction methods 4,20,60 days in the future plot = PlotHelper() prices = PricingData(ticker) if prices.LoadHistory(): prices.TrimToDateRange('1/1/2000', '3/1/2018') print('Loading ' + ticker) for daysForward in [4, 20, 60]: for predictionMethod in range(0, 5): modelDescription = ticker + '_method' + str( predictionMethod) + '_epochs' + str( numberOfLearningPasses) + '_daysforward' + str( daysForward) print('Predicting ' + str(daysForward) + ' days using method ' + modelDescription) prices.PredictPrices(predictionMethod, daysForward, numberOfLearningPasses) predDF = prices.pricePredictions.copy() predDF = predDF.join(prices.GetPriceHistory()) predDF['PercentageDeviation'] = abs( (predDF['Average'] - predDF['estAverage']) / predDF['Average']) averageDeviation = predDF['PercentageDeviation'].tail( round(predDF.shape[0] / 4)).mean( ) #Average of the last 25% to account for training. print('Average deviation: ', averageDeviation * 100, '%') predDF.to_csv(dataFolder + modelDescription + '.csv') plot.PlotDataFrame(predDF[['estAverage', 'Average']], modelDescription, 'Date', 'Price', True, dataFolder + modelDescription) plot.PlotDataFrameDateRange( predDF[['Average', 'estAverage']], None, 160, modelDescription + '_last160ays', 'Date', 'Price', dataFolder + modelDescription + '_last160Days') plot.PlotDataFrameDateRange( predDF[['Average', 'estAverage']], None, 500, modelDescription + '_last500Days', 'Date', 'Price', dataFolder + modelDescription + '_last500Days')
def PredictPrices(prices: PricingData, predictionMethod: int = 0, daysForward: int = 5, numberOfLearningPasses: int = 500): #Procedure to execute a given prediction method: linear projection, LSTM, CNN #Results are exported to the "experiment" sub folder, including a CSV file containing actual and predicted data, and graphs assert (0 <= predictionMethod <= 2) plot = PlotHelper() if predictionMethod == 0: #Linear projection print('Running Linear Projection model predicting ' + str(daysForward) + ' days...') modelDescription = prices.stockTicker + '_Linear_daysforward' + str( daysForward) predDF = prices.GetPriceHistory() predDF['Average'] = (predDF['Open'] + predDF['High'] + predDF['Low'] + predDF['Close']) / 4 d = predDF.index[-1] for i in range( 0, daysForward ): #Add new days to the end for crystal ball predictions predDF.loc[d + BDay(i + 1), 'Average_Predicted'] = 0 predDF['PastSlope'] = predDF['Average'].shift( daysForward) / predDF['Average'].shift(daysForward * 2) predDF['Average_Predicted'] = predDF['Average'].shift( daysForward) * predDF['PastSlope'] predDF['PercentageDeviation'] = abs( (predDF['Average'] - predDF['Average_Predicted']) / predDF['Average']) else: SourceFieldList = ['High', 'Low', 'Open', 'Close'] if predictionMethod == 1: #LSTM learning print('Running LSTM model predicting ' + str(daysForward) + ' days...') SourceFieldList = None UseLSTM = True window_size = 10 modelDescription = prices.stockTicker + '_LSTM' + '_epochs' + str( numberOfLearningPasses) + '_histwin' + str( window_size) + '_daysforward' + str(daysForward) elif predictionMethod == 2: #CNN Learning print('Running CNN model predicting ' + str(daysForward) + ' days...') UseLSTM = False window_size = 16 * daysForward modelDescription = prices.stockTicker + '_CNN' + '_epochs' + str( numberOfLearningPasses) + '_histwin' + str( window_size) + '_daysforward' + str(daysForward) learningModule = StockPredictionNN(baseModelName=prices.stockTicker, UseLSTM=UseLSTM) learningModule.LoadSource(prices.GetPriceHistory(), FieldList=SourceFieldList, window_size=window_size) learningModule.LoadTarget(targetDF=None, prediction_target_days=daysForward) learningModule.MakeBatches(batch_size=32, train_test_split=.93) learningModule.Train(epochs=numberOfLearningPasses) learningModule.Predict(True) predDF = learningModule.GetTrainingResults(True, True) predDF['PercentageDeviation'] = abs( (predDF['Average'] - predDF['Average_Predicted']) / predDF['Average']) averageDeviation = predDF['PercentageDeviation'].tail( round(predDF.shape[0] / 4)).mean() #Average of the last 25% to account for training. print('Average deviation: ', averageDeviation * 100, '%') predDF = predDF.reindex(sorted(predDF.columns), axis=1) #Sort columns alphabetical predDF.to_csv(dataFolder + modelDescription + '.csv') plot.PlotDataFrame(predDF[['Average', 'Average_Predicted']], modelDescription, 'Date', 'Price', True, 'experiment/' + modelDescription) plot.PlotDataFrameDateRange(predDF[['Average', 'Average_Predicted']], None, 160, modelDescription + '_last160ays', 'Date', 'Price', dataFolder + modelDescription + '_last160Days') plot.PlotDataFrameDateRange( predDF[['Average', 'Average_Predicted']], None, 1000, modelDescription + '_last1000ays', 'Date', 'Price', dataFolder + modelDescription + '_last1000Days')
def TrainTickerRaw(ticker: str = '^SPX', UseLSTM: bool = True, prediction_target_days: int = 5, epochs: int = 500, usePercentages: bool = False, hidden_layer_size: int = 512, dropout: bool = True, dropout_rate: float = 0.01, learning_rate: float = 2e-5): plot = PlotHelper() prices = PricingData(ticker) print('Loading ' + ticker) if prices.LoadHistory(True): prices.TrimToDateRange('1/1/2000', '3/1/2018') if usePercentages: prices.ConvertToPercentages( ) #Percentages don't work well I suspect because small errors have a huge impact when you revert back to the original prices and they roll forward else: prices.NormalizePrices() prices.CalculateStats() model = StockPredictionNN(baseModelName=ticker, UseLSTM=UseLSTM) if UseLSTM: window_size = 1 modelDescription = ticker + '_LSTM' modelDescription += '_epochs' + str(epochs) + '_histwin' + str( window_size) + '_daysforward' + str(prediction_target_days) if usePercentages: modelDescription += '_percentages' FieldList = ['Average'] model.LoadSource(sourceDF=prices.GetPriceHistory(), FieldList=FieldList, window_size=window_size) model.LoadTarget(targetDF=None, prediction_target_days=prediction_target_days) model.MakeBatches(batch_size=128, train_test_split=.93) model.BuildModel(layer_count=1, hidden_layer_size=hidden_layer_size, dropout=dropout, dropout_rate=dropout_rate, learning_rate=learning_rate) model.DisplayModel() model.Train(epochs=epochs) model.Predict(True) model.Save() #model.DisplayDataSample() else: #CNN window_size = 16 * prediction_target_days modelDescription = ticker + '_CNN' modelDescription += '_epochs' + str(epochs) + '_histwin' + str( window_size) + '_daysforward' + str(prediction_target_days) if usePercentages: modelDescription += '_percentages' #FieldList = None FieldList = ['High', 'Low', 'Open', 'Close'] model.LoadSource(sourceDF=prices.GetPriceHistory(), FieldList=FieldList, window_size=window_size) model.LoadTarget(targetDF=None, prediction_target_days=prediction_target_days) model.MakeBatches(batch_size=64, train_test_split=.93) model.BuildModel(layer_count=1, hidden_layer_size=hidden_layer_size, dropout=dropout, dropout_rate=dropout_rate, learning_rate=learning_rate) model.DisplayModel() model.Train(epochs=epochs) model.Predict(True) model.Save() if usePercentages: predDF = model.GetTrainingResults(True, True) predDF = predDF.loc[:, ['Average', 'Average_Predicted']] print('Unraveling percentages..') predDF['Average_Predicted'].fillna(0, inplace=True) predDF.iloc[0] = prices.CTPFactor['Average'] for i in range(1, predDF.shape[0]): predDF.iloc[i] = (1 + predDF.iloc[i]) * predDF.iloc[i - 1] print(predDF) predDF['PercentageDeviation'] = abs( (predDF['Average'] - predDF['Average_Predicted']) / predDF['Average']) predDF.to_csv(dataFolder + modelDescription + '.csv') plot.PlotDataFrame(predDF[['Average', 'Average_Predicted']], modelDescription, 'Date', 'Price', True, dataFolder + modelDescription) plot.PlotDataFrameDateRange( predDF[['Average', 'Average_Predicted']], None, 160, modelDescription + '_last160ays', 'Date', 'Price', dataFolder + modelDescription + '_last160Days') plot.PlotDataFrameDateRange( predDF[['Average', 'Average_Predicted']], None, 500, modelDescription + '_last500Days', 'Date', 'Price', dataFolder + modelDescription + '_last500Days') else: model.PredictionResultsSave(modelDescription, True, True) model.PredictionResultsPlot(modelDescription, True, False)