def generateRawPredictionsMP(allModels, joinedData, threadsToUse):
mpEngine = mp.get_context('fork')
def runMod(mod, joinedData):
pred = mod.runModelToday(joinedData)
print(mod.describe(), pred, joinedData.index[-1])
curveTreeDB.storeModelPrediction(mod, pred, joinedData.index[-1])
##ENSURE POPULATED FOR CORRECT PREDICTION STYLE
i = mod.predictionDistance - 1
while i > 0:
pred = mod.runModelToday(joinedData[:-i])
print(mod.describe(), pred, joinedData[:-i].index[-1])
curveTreeDB.storeModelPrediction(mod, pred,
joinedData[:-i].index[-1])
i -= 1
runningP = []
for mod in allModels:
while len(runningP) > threadsToUse:
runningP = dataAck.cycleP(runningP)
p = mpEngine.Process(target=runMod, args=(
mod,
joinedData,
))
p.start()
runningP.append(p)
while len(runningP) > 0:
runningP = dataAck.cycleP(runningP)
return True
def runBackfillMP(mod, joinedData, threadsToUse, backfillDays = 30):
mpEngine = mp.get_context('fork')
i = mod.inputSeries.predictionPeriod - 1 + backfillDays
runningP = []
while i > 0:
while len(runningP) > threadsToUse:
runningP = dataAck.cycleP(runningP)
p = mpEngine.Process(target=runModPredictionBackfill, args=(mod, joinedData[:-i], backfillDays, ))
p.start()
runningP.append(p)
i -= 1
while len(runningP) > 0:
runningP = dataAck.cycleP(runningP)
print("CHECKING AGGREGATE PREDICTIONS")
##STORE AGGREGATE PREDICTIONS
i = mod.inputSeries.predictionPeriod - 1 + backfillDays
allPreds = portfolio.getPredictionsByModel(mod)
while i > 0:
lastDay = joinedData[:-i].index[-1]
todayPredictions = []
for pred in allPreds:
##CHECK IF PREDICTION STILL VALID
if len(joinedData[str(pred["lastDataDayUsed"]):lastDay]) - 1 < pred["predictionLength"] and len(joinedData[str(pred["lastDataDayUsed"]):lastDay]) > 0:##GETS TRADING DAYS SINCE LAST DATA DAY
todayPredictions.append(pred["prediction"])
##SKIP UPLOAD IF NOT ENOUGH PREDICTIONS
print(lastDay, len(todayPredictions))
if len(todayPredictions) == mod.inputSeries.predictionPeriod:
pred = dataAck.computePosition(todayPredictions)
print(mod.describe(), todayPredictions, pred)
portfolio.storeAggregateModelPrediction(mod, pred, lastDay)
i -= 1
def getValidCounts(db):
allTickers = dataAck.getAllTickersPlain()
mpEngine = mp.get_context('fork')
with mpEngine.Manager() as manager:
returnDict = manager.dict()
runningP = []
for ticker in allTickers:
while len(runningP) > 16:
runningP = dataAck.cycleP(runningP)
p = mpEngine.Process(target=getValidModelsByTicker,
args=(
db,
ticker,
returnDict,
))
p.start()
runningP.append(p)
while len(runningP) > 0:
runningP = dataAck.cycleP(runningP)
storedData = {}
for ticker in allTickers:
try:
if returnDict[ticker] is not None:
storedData[ticker] = returnDict[ticker]
except:
continue
return storedData
def createPossiblePortfoliosMP(cleanedPredictions, cleanedReturns, hashToModel, joinedData, threadsToUse):
mpEngine = mp.get_context('fork')
runningP = []
allModels = [hashToModel[item] for item in cleanedPredictions.columns]
modsPerTicker = getAlgosForTicker(allModels)
while True:
selectedAlgorithms = returnSelectTickers(modsPerTicker)
factorToTrade = "VTI"#hashToModel[selectedAlgorithms[random.randint(0, len(selectedAlgorithms) - 1)]].targetTicker
while len(runningP) > threadsToUse:
runningP = dataAck.cycleP(runningP)
time.sleep(3)
portfolioType = types[random.randint(0, len(types) - 1)]
print(factorToTrade, len(selectedAlgorithms), portfolioType)
p = mpEngine.Process(target=performPortfolioPerformanceEstimation, args=(cleanedPredictions[selectedAlgorithms], cleanedReturns[selectedAlgorithms], factorToTrade, portfolioType, hashToModel, joinedData))
p.start()
runningP.append(p)
p = mpEngine.Process(target=performPortfolioPerformanceEstimation, args=(cleanedPredictions[selectedAlgorithms], cleanedReturns[selectedAlgorithms], factorToTrade, "EW", hashToModel, joinedData))
p.start()
runningP.append(p)
p = mpEngine.Process(target=performPortfolioPerformanceEstimation, args=(cleanedPredictions[selectedAlgorithms], cleanedReturns[selectedAlgorithms], factorToTrade, "EW By Ticker", hashToModel, joinedData))
p.start()
runningP.append(p)
def createPossiblePortfoliosMP(cleanedPredictions, cleanedReturns, hashToModel, joinedData, threadsToUse):
mpEngine = mp.get_context('fork')
runningP = []
while True:
selectedAlgorithms = returnSelectAlgos(cleanedReturns.columns)
while len(runningP) > threadsToUse:
runningP = dataAck.cycleP(runningP)
p = mpEngine.Process(target=performPortfolioPerformanceEstimation, args=(cleanedPredictions[selectedAlgorithms], cleanedReturns[selectedAlgorithms], hashToModel, joinedData, ))
p.start()
runningP.append(p)
def generateRawPredictionsMP(allModels, joinedData, threadsToUse):
mpEngine = mp.get_context('fork')
def runMod(mod, joinedData):
pred = dataAck.computePosition([
mod.makeTodayPrediction(
portfolio.prepareDataForModel(mod, joinedData))
])
print(mod.describe(), pred, joinedData.index[-1])
portfolio.storeModelPrediction(mod, pred, joinedData.index[-1])
##ENSURE POPULATED FOR CORRECT PREDICTION STYLE
i = mod.inputSeries.predictionPeriod - 1
while i > 0:
pred = dataAck.computePosition(
[mod.makeTodayPrediction(joinedData[:-i])])
print(mod.describe(), pred, joinedData[:-i].index[-1])
portfolio.storeModelPrediction(mod, pred,
joinedData[:-i].index[-1])
i -= 1
runningP = []
for mod in allModels:
while len(runningP) > threadsToUse:
runningP = dataAck.cycleP(runningP)
p = mpEngine.Process(target=runMod, args=(
mod,
joinedData,
))
p.start()
runningP.append(p)
while len(runningP) > 0:
runningP = dataAck.cycleP(runningP)
return True