def performPortfolioPerformanceEstimation(thisPredictions, thisReturns,
hashToModel, joinedData):
hrpReturns, historicalWeights = portfolioGeneration.\
produceHRPPredictions(thisReturns,\
126, startIndex=None, maxWindowSize=False)
print("COMPUTED HISTORICAL WEIGHTS")
modelsUsed = []
tickersSeen = {}
for modelHash in thisPredictions.columns:
thisModel = hashToModel[modelHash]
modelsUsed.append(thisModel)
print(thisModel.describe(), thisModel.getHash())
if thisModel.targetTicker not in tickersSeen:
tickersSeen[thisModel.targetTicker] = 0
tickersSeen[thisModel.targetTicker] += 1
##STORE MODEL
portfolioHash = storePortfolio(modelsUsed,\
description=str(tickersSeen), benchmark="SPY", portfolioType="HRP FULL")
storeHistoricalAllocations(portfolioHash, \
modelsUsed, historicalWeights, thisPredictions)
portfolioInfo = portfolio.getPortfolioByKey(portfolioHash)
portfolioInfo = {
"key": portfolioInfo.key.name,
"description": portfolioInfo["description"],
"benchmark": portfolioInfo["benchmark"],
"portfolioType": portfolioInfo["portfolioType"],
"startedTrading": portfolioInfo["startedTrading"]
}
print(portfolioInfo)
portfolioData = getDataForPortfolio(portfolioHash,
portfolioInfo["benchmark"], joinedData,
portfolioInfo["startedTrading"])
portfolioGeneration.cachePortfolio(portfolioInfo, portfolioData,
params.AVAILABLE_MODE)
def getDataForPortfolio(portfolioKey, factorToTrade, joinedData,
availableStartDate):
modelHashes = portfolio.getPortfolioModels(portfolioKey)
models = getModelsByKey(modelHashes)
for model in models:
print(model.describe())
##GENERATE RETURNS FOR PORTFOLIO
portfolioAllocations = portfolio.getPortfolioAllocations(portfolioKey)
predsTable = pd.DataFrame([])
weightsTable = pd.DataFrame([])
tickerAllocationsTable = pd.DataFrame([])
scaledTickerAllocationsTable = pd.DataFrame([])
for allocation in portfolioAllocations:
colsAlgo = []
valsAlgo = []
colsAlgoWeight = []
valsAlgoWeight = []
colsTicker = []
valsTicker = []
colsTickerScaled = []
valsTickerScaled = []
for key in allocation:
if key.startswith("ticker_"):
colsTicker.append(key[len("ticker_"):])
valsTicker.append(allocation[key])
if key.startswith("scaled_ticker_"):
colsTickerScaled.append(key[len("scaled_ticker_"):])
valsTickerScaled.append(
abs(allocation[key]) if np.isnan(allocation[key]) ==
False else 0.0)
if key.startswith("algo_") and not key.startswith("algo_weight_"):
colsAlgo.append(key[len("algo_"):])
valsAlgo.append(allocation[key])
if key.startswith("algo_weight_"):
colsAlgoWeight.append(key[len("algo_weight_"):])
valsAlgoWeight.append(allocation[key])
predsTable = pd.concat([
predsTable,
pd.DataFrame([valsAlgo],
index=[allocation["predictionDay"]],
columns=colsAlgo).tz_localize(None)
])
weightsTable = pd.concat([
weightsTable,
pd.DataFrame([valsAlgoWeight],
index=[allocation["predictionDay"]],
columns=colsAlgoWeight).tz_localize(None)
])
tickerAllocationsTable = pd.concat([
tickerAllocationsTable,
pd.DataFrame([valsTicker],
index=[allocation["predictionDay"]],
columns=colsTicker).tz_localize(None)
])
scaledTickerAllocationsTable = pd.concat([
scaledTickerAllocationsTable,
pd.DataFrame([valsTickerScaled],
index=[allocation["predictionDay"]],
columns=colsTickerScaled).tz_localize(None)
])
predsTable = predsTable.sort_index()
weightsTable = weightsTable.sort_index().fillna(0)
tickerAllocationsTable = tickerAllocationsTable.sort_index().fillna(0)
scaledTickerAllocationsTable = scaledTickerAllocationsTable.sort_index(
).fillna(0)
rawTickerPerformance = portfolioGeneration.calculatePerformanceForTable(
tickerAllocationsTable, tickerAllocationsTable.columns, joinedData)
rawAlgoPerformance = pd.DataFrame(
rawTickerPerformance.apply(lambda x: sum(x), axis=1),
columns=["Algo Return Without Commissions"])
tickerPerformance, algoPerformance, algoTransactionCost = portfolioGeneration.calculatePerformanceForAllocations(
tickerAllocationsTable, joinedData)
benchmark = portfolio.getPortfolioByKey(portfolioKey)["benchmark"]
factorReturn = dataAck.getDailyFactorReturn(benchmark, joinedData)
factorReturn.columns = ["Factor Return (" + benchmark + ")"]
algoPerformance.columns = ["Algo Return"]
algoVsBenchmark = factorReturn.join(algoPerformance).fillna(0)
algoVsBenchmark = algoVsBenchmark.join(rawAlgoPerformance).dropna()
tickerAlphaBetas = []
for ticker in tickerAllocationsTable.columns.values:
thisFactorReturn = dataAck.getDailyFactorReturn(ticker, joinedData)
alpha, beta = empyrical.alpha_beta(algoPerformance, thisFactorReturn)
tickerAlphaBetas.append({
"ticker": ticker,
"alpha": alpha * 100,
"beta": beta
})
##GET SCALED PERFORMANCE [FULL CAPITAL USED EACH DAY]
rawTickerPerformanceScaled = portfolioGeneration.calculatePerformanceForTable(
scaledTickerAllocationsTable, scaledTickerAllocationsTable.columns,
joinedData)
rawAlgoPerformanceScaled = pd.DataFrame(
rawTickerPerformanceScaled.apply(lambda x: sum(x), axis=1),
columns=["Algo Return Without Commissions"])
unused, algoPerformanceScaled, algoTransactionCostScaled = portfolioGeneration.calculatePerformanceForAllocations(
scaledTickerAllocationsTable, joinedData)
algoPerformanceScaled.columns = ["Algo Return"]
algoVsBenchmarkScaled = factorReturn.join(algoPerformanceScaled).fillna(0)
algoVsBenchmarkScaled = algoVsBenchmarkScaled.join(
rawAlgoPerformanceScaled).dropna()
##FORM HASH TO TICKER
hashToTicker = {}
for model in models:
hashToTicker[model.getHash()] = model.targetTicker
print(hashToTicker)
individualAlgoPerformance = portfolioGeneration.calculatePerformanceForTable(
predsTable,
[hashToTicker[modelHash] for modelHash in predsTable.columns],
joinedData)
##CONVERT TO USABLE OBJECTS
tickerCols, tickerRows = portfolioGeneration.convertTableToJSON(
empyrical.cum_returns(tickerPerformance))
tickerAllocationsCols, tickerAllocationsRows = portfolioGeneration.convertTableToJSON(
tickerAllocationsTable[-10:])
algoCols, algoRows = portfolioGeneration.convertTableToJSON(
empyrical.cum_returns(algoPerformance))
algoVsBenchmarkCols, algoVsBenchmarkRows = portfolioGeneration.convertTableToJSON(
empyrical.cum_returns(algoVsBenchmark))
individualAlgoPerformanceCols, individualAlgoPerformanceRows = portfolioGeneration.convertTableToJSON(
empyrical.cum_returns(individualAlgoPerformance))
scaledAllocationCols, scaledAllocationRows = portfolioGeneration.convertTableToJSON(
scaledTickerAllocationsTable)
weightsCols, weightsRows = portfolioGeneration.convertTableToJSON(
weightsTable)
alpha, beta = empyrical.alpha_beta(algoPerformance, factorReturn)
recentAlpha, recentBeta = empyrical.alpha_beta(algoPerformance[-100:],
factorReturn[-100:])
recentSharpe = empyrical.sharpe_ratio(algoPerformance[-100:])
recentReturn = empyrical.cum_returns(
algoPerformance[-100:]).values[-1][0] * 100
algoVsBenchmarkColsRecent, algoVsBenchmarkRowsRecent = portfolioGeneration.convertTableToJSON(
empyrical.cum_returns(algoVsBenchmark[-100:]))
commissionCols, commissionRows = portfolioGeneration.convertTableToJSON(
algoTransactionCost)
algoVsBenchmarkScaledCols, algoVsBenchmarkScaledRows = portfolioGeneration.convertTableToJSON(
empyrical.cum_returns(algoVsBenchmarkScaled))
commissionScaledCols, commissionScaledRows = portfolioGeneration.convertTableToJSON(
algoTransactionCostScaled)
scaledSharpe = empyrical.sharpe_ratio(algoPerformanceScaled)
scaledReturn = empyrical.annual_return(algoPerformanceScaled)[0] * 100
scaledVolatility = empyrical.annual_volatility(algoPerformanceScaled) * 100
scaledAlpha, scaledBeta = empyrical.alpha_beta(algoPerformanceScaled,
factorReturn)
algoVsBenchmarkScaledColsRecent, algoVsBenchmarkScaledRowsRecent = portfolioGeneration.convertTableToJSON(
empyrical.cum_returns(algoVsBenchmarkScaled[-100:]))
scaledSharpeRecent = empyrical.sharpe_ratio(algoPerformanceScaled[-100:])
scaledReturnRecent = empyrical.annual_return(
algoPerformanceScaled[-100:])[0] * 100
scaledVolatilityRecent = empyrical.annual_volatility(
algoPerformanceScaled[-100:]) * 100
scaledAlphaRecent, scaledBetaRecent = empyrical.alpha_beta(
algoPerformanceScaled[-100:], factorReturn[-100:])
if len(algoPerformance[availableStartDate:]) > 0:
##NORMAL
availableAlpha, availableBeta = empyrical.alpha_beta(
algoPerformance[availableStartDate:],
factorReturn[availableStartDate:])
availableAlpha = availableAlpha * 100
availableSharpe = empyrical.sharpe_ratio(
algoPerformance[availableStartDate:])
availableReturn = empyrical.cum_returns(
algoPerformance[availableStartDate:]).values[-1][0] * 100
algoVsBenchmarkColsAvailable, algoVsBenchmarkRowsAvailable = portfolioGeneration.convertTableToJSON(
empyrical.cum_returns(algoVsBenchmark[availableStartDate:]))
##SCALED
availableAlphaScaled, availableBetaScaled = empyrical.alpha_beta(
algoPerformanceScaled[availableStartDate:],
factorReturn[availableStartDate:])
availableAlphaScaled = availableAlphaScaled * 100
availableSharpeScaled = empyrical.sharpe_ratio(
algoPerformanceScaled[availableStartDate:])
availableReturnScaled = empyrical.cum_returns(
algoPerformanceScaled[availableStartDate:]).values[-1][0] * 100
algoVsBenchmarkColsAvailableScaled, algoVsBenchmarkRowsAvailableScaled = portfolioGeneration.convertTableToJSON(
empyrical.cum_returns(algoVsBenchmarkScaled[availableStartDate:]))
else:
#NORMAL
availableAlpha, availableBeta = ("NaN", "NaN")
availableSharpe = "NaN"
availableReturn = "NaN"
algoVsBenchmarkColsAvailable, algoVsBenchmarkRowsAvailable = ([], [])
#SCALED
availableAlphaScaled, availableBetaScaled = ("NaN", "NaN")
availableSharpeScaled = "NaN"
availableReturnScaled = "NaN"
algoVsBenchmarkColsAvailableScaled, algoVsBenchmarkRowsAvailableScaled = (
[], [])
return {
"tickerCols":
json.dumps(tickerCols),
"tickerRows":
json.dumps(tickerRows),
"tickerAllocationsCols":
json.dumps(tickerAllocationsCols),
"tickerAllocationsRows":
json.dumps(tickerAllocationsRows),
"algoCols":
json.dumps(algoCols),
"algoRows":
json.dumps(algoRows),
"tickerCols":
json.dumps(tickerCols),
"tickerRows":
json.dumps(tickerRows),
"algoVsBenchmarkCols":
json.dumps(algoVsBenchmarkCols),
"algoVsBenchmarkRows":
json.dumps(algoVsBenchmarkRows),
"individualAlgoPerformanceCols":
json.dumps(individualAlgoPerformanceCols),
"individualAlgoPerformanceRows":
json.dumps(individualAlgoPerformanceRows),
"scaledAllocationCols":
json.dumps(scaledAllocationCols),
"scaledAllocationRows":
json.dumps(scaledAllocationRows),
"weightsCols":
json.dumps(weightsCols),
"weightsRows":
json.dumps(weightsRows),
"algoSharpe":
empyrical.sharpe_ratio(algoPerformance),
"alpha":
alpha * 100,
"beta":
beta,
"annualReturn":
empyrical.annual_return(algoPerformance)[0] * 100,
"annualVolatility":
empyrical.annual_volatility(algoPerformance) * 100,
"recentSharpe":
recentSharpe,
"recentReturn":
recentReturn,
"recentAlpha":
recentAlpha * 100,
"recentBeta":
recentBeta,
"algoVsBenchmarkColsRecent":
json.dumps(algoVsBenchmarkColsRecent),
"algoVsBenchmarkRowsRecent":
json.dumps(algoVsBenchmarkRowsRecent),
"commissionCols":
json.dumps(commissionCols),
"commissionRows":
json.dumps(commissionRows),
"tickerAlphaBetas":
tickerAlphaBetas,
"availableAlpha":
availableAlpha,
"availableBeta":
availableBeta,
"availableSharpe":
availableSharpe,
"availableReturn":
availableReturn,
"algoVsBenchmarkColsAvailable":
json.dumps(algoVsBenchmarkColsAvailable),
"algoVsBenchmarkRowsAvailable":
json.dumps(algoVsBenchmarkRowsAvailable),
"algoVsBenchmarkScaledCols":
json.dumps(algoVsBenchmarkScaledCols),
"algoVsBenchmarkScaledRows":
json.dumps(algoVsBenchmarkScaledRows),
"commissionScaledCols":
json.dumps(commissionScaledCols),
"commissionScaledRows":
json.dumps(commissionScaledRows),
"scaledReturn":
scaledReturn,
"scaledSharpe":
scaledSharpe,
"scaledVolatility":
scaledVolatility,
"scaledAlpha":
scaledAlpha * 100,
"scaledBeta":
scaledBeta,
"algoVsBenchmarkScaledColsRecent":
json.dumps(algoVsBenchmarkScaledColsRecent),
"algoVsBenchmarkScaledRowsRecent":
json.dumps(algoVsBenchmarkScaledRowsRecent),
"scaledReturnRecent":
scaledReturnRecent,
"scaledVolatilityRecent":
scaledVolatilityRecent,
"scaledAlphaRecent":
scaledAlphaRecent * 100,
"scaledBetaRecent":
scaledBetaRecent,
"scaledSharpeRecent":
scaledSharpeRecent,
"availableAlphaScaled":
availableAlphaScaled,
"availableBetaScaled":
availableBetaScaled,
"availableSharpeScaled":
availableSharpeScaled,
"availableReturnScaled":
availableReturnScaled,
"algoVsBenchmarkColsAvailableScaled":
json.dumps(algoVsBenchmarkColsAvailableScaled),
"algoVsBenchmarkRowsAvailableScaled":
json.dumps(algoVsBenchmarkRowsAvailableScaled),
}
def getDataForPortfolio(portfolioKey):
models = portfolio.getModelsByKey(portfolio.getPortfolioModels(portfolioKey))
##DOWNLOAD REQUIRED DATA FOR TARGET TICKERS
tickersRequired = []
for mod in models:
print(mod.describe())
if mod.inputSeries.targetTicker not in tickersRequired:
tickersRequired.append(mod.inputSeries.targetTicker)
pulledData, validTickers = dataAck.downloadTickerData(tickersRequired)
joinedData = dataAck.joinDatasets([pulledData[ticker] for ticker in pulledData])
##GENERATE RETURNS FOR PORTFOLIO
portfolioAllocations = portfolio.getPortfolioAllocations(portfolioKey)
predsTable = pd.DataFrame([])
weightsTable = pd.DataFrame([])
tickerAllocationsTable = pd.DataFrame([])
scaledTickerAllocationsTable = pd.DataFrame([])
for allocation in portfolioAllocations:
colsAlgo = []
valsAlgo = []
colsAlgoWeight = []
valsAlgoWeight = []
colsTicker = []
valsTicker = []
colsTickerScaled = []
valsTickerScaled = []
for key in allocation:
if key.startswith("ticker_"):
colsTicker.append(key[len("ticker_"):])
valsTicker.append(allocation[key])
if key.startswith("scaled_ticker_"):
colsTickerScaled.append(key[len("scaled_ticker_"):])
valsTickerScaled.append(allocation[key])
if key.startswith("algo_") and not key.startswith("algo_weight_"):
colsAlgo.append(key[len("algo_"):])
valsAlgo.append(allocation[key])
if key.startswith("algo_weight_"):
colsAlgoWeight.append(key[len("algo_weight_"):])
valsAlgoWeight.append(allocation[key])
predsTable = pd.concat([predsTable, pd.DataFrame([valsAlgo], index = [allocation["predictionDay"]], columns=colsAlgo).tz_localize(None)])
weightsTable = pd.concat([weightsTable, pd.DataFrame([valsAlgoWeight], index = [allocation["predictionDay"]], columns=colsAlgoWeight).tz_localize(None)])
tickerAllocationsTable = pd.concat([tickerAllocationsTable, pd.DataFrame([valsTicker], index = [allocation["predictionDay"]], columns=colsTicker).tz_localize(None)])
scaledTickerAllocationsTable = pd.concat([scaledTickerAllocationsTable, pd.DataFrame([valsTickerScaled], index = [allocation["predictionDay"]], columns=colsTickerScaled).tz_localize(None)])
predsTable = predsTable.sort_index()
weightsTable = weightsTable.sort_index()
tickerAllocationsTable = tickerAllocationsTable.sort_index()
scaledTickerAllocationsTable = scaledTickerAllocationsTable.sort_index()
tickerPerformance = calculatePerformanceForTable(tickerAllocationsTable, tickerAllocationsTable.columns, joinedData)
algoPerformance = pd.DataFrame(tickerPerformance.apply(lambda x:sum(x), axis=1), columns=["Algo Return"])
benchmark = portfolio.getPortfolioByKey(portfolioKey)["benchmark"]
factorReturn = dataAck.getDailyFactorReturn(benchmark, joinedData)
factorReturn.columns = ["Factor Return (" + benchmark + ")"]
algoVsBenchmark = algoPerformance.join(factorReturn).dropna()
##FORM HASH TO TICKER
hashToTicker = {}
for model in models:
hashToTicker[portfolio.getModelHash(model)] = model.inputSeries.targetTicker
individualAlgoPerformance = calculatePerformanceForTable(predsTable,[hashToTicker[modelHash] for modelHash in predsTable.columns], joinedData)
return json.dumps(convertTableToJSON(empyrical.cum_returns(tickerPerformance))),\
json.dumps(convertTableToJSON(empyrical.cum_returns(algoPerformance))),\
json.dumps(convertTableToJSON(empyrical.cum_returns(algoVsBenchmark))),\
json.dumps(convertTableToJSON(empyrical.cum_returns(individualAlgoPerformance))),\
# In[42]:
## UPDATE CACHE FOR TRADING PORTFOLIOS
print("****UPDATING ALL PORTFOLIO CACHES****")
for mode in [
params.AVAILABLE_MODE, params.PAPER_TRADING_MODE,
params.TRADING_MODE
]:
print("MODE", mode)
portfolioInfos = []
downloadedPortfolioInfo = portfolioGeneration.getTradingPortfolioHashes(
mode, includeDates=True)
for tradingPortfolio in downloadedPortfolioInfo:
portfolioHash = tradingPortfolio
portfolioInfo = portfolio.getPortfolioByKey(portfolioHash)
portfolioInfo = {
"key": portfolioInfo.key.name,
"description": portfolioInfo["description"],
"benchmark": portfolioInfo["benchmark"],
"portfolioType": portfolioInfo["portfolioType"],
"startedTrading": downloadedPortfolioInfo[portfolioHash]
}
print(portfolioInfo)
portfolioInfos.append(portfolioInfo)
##GET ALL BENCHMARKS
benchmarksNeeded = []
print(joinedData.columns.values)
for info in portfolioInfos:
print(info["benchmark"])
def getNetAllocationAcrossPortfolios():
portfolioInfos = []
downloadedPortfolioInfo = portfolioGeneration.getTradingPortfolioHashes(
params.TRADING_MODE, includeDates=True)
for tradingPortfolio in downloadedPortfolioInfo:
portfolioHash = tradingPortfolio
portfolioInfo = portfolio.getPortfolioByKey(portfolioHash)
portfolioInfo = {
"key": portfolioInfo.key.name,
"description": portfolioInfo["description"],
"benchmark": portfolioInfo["benchmark"],
"portfolioType": portfolioInfo["portfolioType"],
"startedTrading": downloadedPortfolioInfo[portfolioHash]
}
print(portfolioInfo)
portfolioInfos.append(portfolioInfo)
if len(portfolioInfos) == 0:
return None, None
#DAILY ALLOCATIONS MUST ONLY GET ALLOCATIONS FOR PORTFOLIO AFTER START DATE FOR PORTFOLIO
totalDesired = {}
historicalAllocations = {}
realizedAllocations = {}
for tradingPortfolio in portfolioInfos:
portfolioHash = tradingPortfolio["key"]
allAllocations = portfolio.getPortfolioAllocations(portfolioHash)
for item in allAllocations:
netPosition = {}
for key in item:
if key.startswith("ticker_"):
netPosition[key[len("ticker_"):]] = item[key]
##REALIZED ALLOCATIONS
if item["predictionDay"] >= tradingPortfolio["startedTrading"]:
if item["predictionDay"] not in realizedAllocations:
realizedAllocations[item["predictionDay"]] = {}
for ticker in netPosition:
if ticker not in realizedAllocations[
item["predictionDay"]]:
realizedAllocations[
item["predictionDay"]][ticker] = 0.0
realizedAllocations[
item["predictionDay"]][ticker] += netPosition[ticker]
if item["predictionDay"] not in historicalAllocations:
historicalAllocations[item["predictionDay"]] = {}
for ticker in netPosition:
if ticker not in historicalAllocations[item["predictionDay"]]:
historicalAllocations[item["predictionDay"]][ticker] = 0.0
historicalAllocations[
item["predictionDay"]][ticker] += netPosition[ticker]
##CLEANUP ALLOCATIONS TO SCALE TO CAPITAL 1
realizedAllocations = createAllocationsTable(
cleanupAllocations(realizedAllocations))
historicalAllocations = createAllocationsTable(
cleanupAllocations(historicalAllocations))
return historicalAllocations, realizedAllocations