def calculatePerformanceForTable(table, tickerOrder, joinedData):
aggregatePerformance = None
for i in range(len(tickerOrder)):
dailyFactorReturn = dataAck.getDailyFactorReturn(tickerOrder[i], joinedData)
thisPerformance = table[[table.columns[i]]].join(dailyFactorReturn).apply(lambda x:x[0] * x[1], axis=1)
thisPerformance = pd.DataFrame(thisPerformance, columns=[table.columns[i]])
if aggregatePerformance is None:
aggregatePerformance = thisPerformance
else:
aggregatePerformance = aggregatePerformance.join(thisPerformance)
return aggregatePerformance.dropna()
def runModelHistorical(self, dataOfInterest):
dailyFactorReturn = dataAck.getDailyFactorReturn(
self.targetTicker, dataOfInterest)
predictions = pd.DataFrame(dailyFactorReturn.apply(
lambda x: -1.0 if self.position == 0 else 1.0, axis=1),
columns=["Predictions"]).dropna()
transformedPreds = predictions.join(dailyFactorReturn).dropna()
returnStream = pd.DataFrame(transformedPreds.apply(
lambda x: x[0] * x[1], axis=1),
columns=["Algo Return"])
return returnStream, dailyFactorReturn, predictions, returnStream, predictions
def runModelHistorical(self, dataOfInterest):
##RAW PREDICTIONS ARE PREDS 0->1.0
returnStream, factorReturn, predictions1, slippageAdjustedReturn, rawPredictions1 = self.obj1.runModelHistorical(
dataOfInterest)
returnStream, factorReturn, predictions2, slippageAdjustedReturn, rawPredictions2 = self.obj2.runModelHistorical(
dataOfInterest)
positions = predictions1.join(predictions2, rsuffix="2").dropna()
# print(rawPredictions)
#averagePredictions
positionsTable = pd.DataFrame(
positions.apply(lambda x: self.combinePredictions(x),
axis=1,
raw=True))
print("POSITIONS TABLE")
print(positionsTable)
##PREDICTIONS COMBINED AS 0, 0.5, 1 where
returnStream = None
factorReturn = None
predictions = None
slippageAdjustedReturn = None
positionsTable.columns = ["Positions"]
positionsTable = positionsTable.dropna()
rawPositions = positionsTable
##AVERAGE...A LOT OF SUBTLETY IN STRENGTH OF PREDICTION
dailyFactorReturn = dataAck.getDailyFactorReturn(
self.targetTicker, dataOfInterest)
transformedPositions = positionsTable.join(dailyFactorReturn).dropna()
returnStream = pd.DataFrame(transformedPositions.apply(
lambda x: x[0] * x[1], axis=1),
columns=["Algo Return"])
factorReturn = pd.DataFrame(transformedPositions[["Factor Return"]])
positions = pd.DataFrame(transformedPositions[["Positions"]])
estimatedSlippageLoss = portfolioGeneration.estimateTransactionCost(
positions)
estimatedSlippageLoss.columns = returnStream.columns
slippageAdjustedReturn = (returnStream -
estimatedSlippageLoss).dropna()
return returnStream, factorReturn, positions, slippageAdjustedReturn, rawPositions
def generateAggregateReturnsPredictions(allModels, joinedData):
aggregateReturns = None
aggregatePredictions = None
for model in allModels:
preds = downloadAggregatePredictions(model).tz_localize(None)
dailyFactorReturn = dataAck.getDailyFactorReturn(
model.targetTicker, joinedData)
transformedPreds = preds.join(dailyFactorReturn).dropna()
returnStream = pd.DataFrame(transformedPreds.apply(
lambda x: x[0] * x[1], axis=1),
columns=[model.getHash()])
preds.columns = [model.getHash()]
if aggregateReturns is None:
aggregateReturns = returnStream
aggregatePredictions = preds
else:
aggregateReturns = aggregateReturns.join(returnStream)
aggregatePredictions = aggregatePredictions.join(preds)
return aggregateReturns, aggregatePredictions
def computeReturnsForUniqueModelsCache(treeModels, factorToTrade):
tickersRequired = []
for mod in treeModels:
if mod.targetTicker not in tickersRequired:
tickersRequired.append(mod.targetTicker)
if factorToTrade not in tickersRequired:
tickersRequired.append(factorToTrade)
print(tickersRequired)
pulledData, validTickers = dataAck.downloadTickerData(tickersRequired)
joinedData = dataAck.joinDatasets(
[pulledData[ticker] for ticker in pulledData])
modelReturns, modelPredictions, modelSlippageReturns, cleanedModels = generateAllReturnsFromCache(
treeModels)
return cleanedModels, modelReturns, modelPredictions, modelSlippageReturns, modelReturns.join(
dataAck.getDailyFactorReturn(factorToTrade,
joinedData)).dropna(), joinedData
def getDataForPortfolio(portfolioKey, factorToTrade, joinedData,
availableStartDate):
modelHashes = portfolio.getPortfolioModels(portfolioKey)
models = getModelsByKey(modelHashes)
for model in models:
print(model.describe())
##GENERATE RETURNS FOR PORTFOLIO
portfolioAllocations = portfolio.getPortfolioAllocations(portfolioKey)
predsTable = pd.DataFrame([])
weightsTable = pd.DataFrame([])
tickerAllocationsTable = pd.DataFrame([])
scaledTickerAllocationsTable = pd.DataFrame([])
for allocation in portfolioAllocations:
colsAlgo = []
valsAlgo = []
colsAlgoWeight = []
valsAlgoWeight = []
colsTicker = []
valsTicker = []
colsTickerScaled = []
valsTickerScaled = []
for key in allocation:
if key.startswith("ticker_"):
colsTicker.append(key[len("ticker_"):])
valsTicker.append(allocation[key])
if key.startswith("scaled_ticker_"):
colsTickerScaled.append(key[len("scaled_ticker_"):])
valsTickerScaled.append(
abs(allocation[key]) if np.isnan(allocation[key]) ==
False else 0.0)
if key.startswith("algo_") and not key.startswith("algo_weight_"):
colsAlgo.append(key[len("algo_"):])
valsAlgo.append(allocation[key])
if key.startswith("algo_weight_"):
colsAlgoWeight.append(key[len("algo_weight_"):])
valsAlgoWeight.append(allocation[key])
predsTable = pd.concat([
predsTable,
pd.DataFrame([valsAlgo],
index=[allocation["predictionDay"]],
columns=colsAlgo).tz_localize(None)
])
weightsTable = pd.concat([
weightsTable,
pd.DataFrame([valsAlgoWeight],
index=[allocation["predictionDay"]],
columns=colsAlgoWeight).tz_localize(None)
])
tickerAllocationsTable = pd.concat([
tickerAllocationsTable,
pd.DataFrame([valsTicker],
index=[allocation["predictionDay"]],
columns=colsTicker).tz_localize(None)
])
scaledTickerAllocationsTable = pd.concat([
scaledTickerAllocationsTable,
pd.DataFrame([valsTickerScaled],
index=[allocation["predictionDay"]],
columns=colsTickerScaled).tz_localize(None)
])
predsTable = predsTable.sort_index()
weightsTable = weightsTable.sort_index().fillna(0)
tickerAllocationsTable = tickerAllocationsTable.sort_index().fillna(0)
scaledTickerAllocationsTable = scaledTickerAllocationsTable.sort_index(
).fillna(0)
rawTickerPerformance = portfolioGeneration.calculatePerformanceForTable(
tickerAllocationsTable, tickerAllocationsTable.columns, joinedData)
rawAlgoPerformance = pd.DataFrame(
rawTickerPerformance.apply(lambda x: sum(x), axis=1),
columns=["Algo Return Without Commissions"])
tickerPerformance, algoPerformance, algoTransactionCost = portfolioGeneration.calculatePerformanceForAllocations(
tickerAllocationsTable, joinedData)
benchmark = portfolio.getPortfolioByKey(portfolioKey)["benchmark"]
factorReturn = dataAck.getDailyFactorReturn(benchmark, joinedData)
factorReturn.columns = ["Factor Return (" + benchmark + ")"]
algoPerformance.columns = ["Algo Return"]
algoVsBenchmark = factorReturn.join(algoPerformance).fillna(0)
algoVsBenchmark = algoVsBenchmark.join(rawAlgoPerformance).dropna()
tickerAlphaBetas = []
for ticker in tickerAllocationsTable.columns.values:
thisFactorReturn = dataAck.getDailyFactorReturn(ticker, joinedData)
alpha, beta = empyrical.alpha_beta(algoPerformance, thisFactorReturn)
tickerAlphaBetas.append({
"ticker": ticker,
"alpha": alpha * 100,
"beta": beta
})
##GET SCALED PERFORMANCE [FULL CAPITAL USED EACH DAY]
rawTickerPerformanceScaled = portfolioGeneration.calculatePerformanceForTable(
scaledTickerAllocationsTable, scaledTickerAllocationsTable.columns,
joinedData)
rawAlgoPerformanceScaled = pd.DataFrame(
rawTickerPerformanceScaled.apply(lambda x: sum(x), axis=1),
columns=["Algo Return Without Commissions"])
unused, algoPerformanceScaled, algoTransactionCostScaled = portfolioGeneration.calculatePerformanceForAllocations(
scaledTickerAllocationsTable, joinedData)
algoPerformanceScaled.columns = ["Algo Return"]
algoVsBenchmarkScaled = factorReturn.join(algoPerformanceScaled).fillna(0)
algoVsBenchmarkScaled = algoVsBenchmarkScaled.join(
rawAlgoPerformanceScaled).dropna()
##FORM HASH TO TICKER
hashToTicker = {}
for model in models:
hashToTicker[model.getHash()] = model.targetTicker
print(hashToTicker)
individualAlgoPerformance = portfolioGeneration.calculatePerformanceForTable(
predsTable,
[hashToTicker[modelHash] for modelHash in predsTable.columns],
joinedData)
##CONVERT TO USABLE OBJECTS
tickerCols, tickerRows = portfolioGeneration.convertTableToJSON(
empyrical.cum_returns(tickerPerformance))
tickerAllocationsCols, tickerAllocationsRows = portfolioGeneration.convertTableToJSON(
tickerAllocationsTable[-10:])
algoCols, algoRows = portfolioGeneration.convertTableToJSON(
empyrical.cum_returns(algoPerformance))
algoVsBenchmarkCols, algoVsBenchmarkRows = portfolioGeneration.convertTableToJSON(
empyrical.cum_returns(algoVsBenchmark))
individualAlgoPerformanceCols, individualAlgoPerformanceRows = portfolioGeneration.convertTableToJSON(
empyrical.cum_returns(individualAlgoPerformance))
scaledAllocationCols, scaledAllocationRows = portfolioGeneration.convertTableToJSON(
scaledTickerAllocationsTable)
weightsCols, weightsRows = portfolioGeneration.convertTableToJSON(
weightsTable)
alpha, beta = empyrical.alpha_beta(algoPerformance, factorReturn)
recentAlpha, recentBeta = empyrical.alpha_beta(algoPerformance[-100:],
factorReturn[-100:])
recentSharpe = empyrical.sharpe_ratio(algoPerformance[-100:])
recentReturn = empyrical.cum_returns(
algoPerformance[-100:]).values[-1][0] * 100
algoVsBenchmarkColsRecent, algoVsBenchmarkRowsRecent = portfolioGeneration.convertTableToJSON(
empyrical.cum_returns(algoVsBenchmark[-100:]))
commissionCols, commissionRows = portfolioGeneration.convertTableToJSON(
algoTransactionCost)
algoVsBenchmarkScaledCols, algoVsBenchmarkScaledRows = portfolioGeneration.convertTableToJSON(
empyrical.cum_returns(algoVsBenchmarkScaled))
commissionScaledCols, commissionScaledRows = portfolioGeneration.convertTableToJSON(
algoTransactionCostScaled)
scaledSharpe = empyrical.sharpe_ratio(algoPerformanceScaled)
scaledReturn = empyrical.annual_return(algoPerformanceScaled)[0] * 100
scaledVolatility = empyrical.annual_volatility(algoPerformanceScaled) * 100
scaledAlpha, scaledBeta = empyrical.alpha_beta(algoPerformanceScaled,
factorReturn)
algoVsBenchmarkScaledColsRecent, algoVsBenchmarkScaledRowsRecent = portfolioGeneration.convertTableToJSON(
empyrical.cum_returns(algoVsBenchmarkScaled[-100:]))
scaledSharpeRecent = empyrical.sharpe_ratio(algoPerformanceScaled[-100:])
scaledReturnRecent = empyrical.annual_return(
algoPerformanceScaled[-100:])[0] * 100
scaledVolatilityRecent = empyrical.annual_volatility(
algoPerformanceScaled[-100:]) * 100
scaledAlphaRecent, scaledBetaRecent = empyrical.alpha_beta(
algoPerformanceScaled[-100:], factorReturn[-100:])
if len(algoPerformance[availableStartDate:]) > 0:
##NORMAL
availableAlpha, availableBeta = empyrical.alpha_beta(
algoPerformance[availableStartDate:],
factorReturn[availableStartDate:])
availableAlpha = availableAlpha * 100
availableSharpe = empyrical.sharpe_ratio(
algoPerformance[availableStartDate:])
availableReturn = empyrical.cum_returns(
algoPerformance[availableStartDate:]).values[-1][0] * 100
algoVsBenchmarkColsAvailable, algoVsBenchmarkRowsAvailable = portfolioGeneration.convertTableToJSON(
empyrical.cum_returns(algoVsBenchmark[availableStartDate:]))
##SCALED
availableAlphaScaled, availableBetaScaled = empyrical.alpha_beta(
algoPerformanceScaled[availableStartDate:],
factorReturn[availableStartDate:])
availableAlphaScaled = availableAlphaScaled * 100
availableSharpeScaled = empyrical.sharpe_ratio(
algoPerformanceScaled[availableStartDate:])
availableReturnScaled = empyrical.cum_returns(
algoPerformanceScaled[availableStartDate:]).values[-1][0] * 100
algoVsBenchmarkColsAvailableScaled, algoVsBenchmarkRowsAvailableScaled = portfolioGeneration.convertTableToJSON(
empyrical.cum_returns(algoVsBenchmarkScaled[availableStartDate:]))
else:
#NORMAL
availableAlpha, availableBeta = ("NaN", "NaN")
availableSharpe = "NaN"
availableReturn = "NaN"
algoVsBenchmarkColsAvailable, algoVsBenchmarkRowsAvailable = ([], [])
#SCALED
availableAlphaScaled, availableBetaScaled = ("NaN", "NaN")
availableSharpeScaled = "NaN"
availableReturnScaled = "NaN"
algoVsBenchmarkColsAvailableScaled, algoVsBenchmarkRowsAvailableScaled = (
[], [])
return {
"tickerCols":
json.dumps(tickerCols),
"tickerRows":
json.dumps(tickerRows),
"tickerAllocationsCols":
json.dumps(tickerAllocationsCols),
"tickerAllocationsRows":
json.dumps(tickerAllocationsRows),
"algoCols":
json.dumps(algoCols),
"algoRows":
json.dumps(algoRows),
"tickerCols":
json.dumps(tickerCols),
"tickerRows":
json.dumps(tickerRows),
"algoVsBenchmarkCols":
json.dumps(algoVsBenchmarkCols),
"algoVsBenchmarkRows":
json.dumps(algoVsBenchmarkRows),
"individualAlgoPerformanceCols":
json.dumps(individualAlgoPerformanceCols),
"individualAlgoPerformanceRows":
json.dumps(individualAlgoPerformanceRows),
"scaledAllocationCols":
json.dumps(scaledAllocationCols),
"scaledAllocationRows":
json.dumps(scaledAllocationRows),
"weightsCols":
json.dumps(weightsCols),
"weightsRows":
json.dumps(weightsRows),
"algoSharpe":
empyrical.sharpe_ratio(algoPerformance),
"alpha":
alpha * 100,
"beta":
beta,
"annualReturn":
empyrical.annual_return(algoPerformance)[0] * 100,
"annualVolatility":
empyrical.annual_volatility(algoPerformance) * 100,
"recentSharpe":
recentSharpe,
"recentReturn":
recentReturn,
"recentAlpha":
recentAlpha * 100,
"recentBeta":
recentBeta,
"algoVsBenchmarkColsRecent":
json.dumps(algoVsBenchmarkColsRecent),
"algoVsBenchmarkRowsRecent":
json.dumps(algoVsBenchmarkRowsRecent),
"commissionCols":
json.dumps(commissionCols),
"commissionRows":
json.dumps(commissionRows),
"tickerAlphaBetas":
tickerAlphaBetas,
"availableAlpha":
availableAlpha,
"availableBeta":
availableBeta,
"availableSharpe":
availableSharpe,
"availableReturn":
availableReturn,
"algoVsBenchmarkColsAvailable":
json.dumps(algoVsBenchmarkColsAvailable),
"algoVsBenchmarkRowsAvailable":
json.dumps(algoVsBenchmarkRowsAvailable),
"algoVsBenchmarkScaledCols":
json.dumps(algoVsBenchmarkScaledCols),
"algoVsBenchmarkScaledRows":
json.dumps(algoVsBenchmarkScaledRows),
"commissionScaledCols":
json.dumps(commissionScaledCols),
"commissionScaledRows":
json.dumps(commissionScaledRows),
"scaledReturn":
scaledReturn,
"scaledSharpe":
scaledSharpe,
"scaledVolatility":
scaledVolatility,
"scaledAlpha":
scaledAlpha * 100,
"scaledBeta":
scaledBeta,
"algoVsBenchmarkScaledColsRecent":
json.dumps(algoVsBenchmarkScaledColsRecent),
"algoVsBenchmarkScaledRowsRecent":
json.dumps(algoVsBenchmarkScaledRowsRecent),
"scaledReturnRecent":
scaledReturnRecent,
"scaledVolatilityRecent":
scaledVolatilityRecent,
"scaledAlphaRecent":
scaledAlphaRecent * 100,
"scaledBetaRecent":
scaledBetaRecent,
"scaledSharpeRecent":
scaledSharpeRecent,
"availableAlphaScaled":
availableAlphaScaled,
"availableBetaScaled":
availableBetaScaled,
"availableSharpeScaled":
availableSharpeScaled,
"availableReturnScaled":
availableReturnScaled,
"algoVsBenchmarkColsAvailableScaled":
json.dumps(algoVsBenchmarkColsAvailableScaled),
"algoVsBenchmarkRowsAvailableScaled":
json.dumps(algoVsBenchmarkRowsAvailableScaled),
}
def getLimitedDataForPortfolio(historicalWeights, historicalPredictions, modelsUsed, factorToTrade, joinedData):
normalTickerAllocationsTable, scaledTickerAllocationsTable = historicalWeightsToTickerAllocations(historicalWeights, historicalPredictions, modelsUsed)
# capitalUsed = pd.DataFrame(normalTickerAllocationsTable.apply(lambda x: sum([abs(item) for item in x]), axis=1))
# print(capitalUsed)
tickerAllocationsTable = scaledTickerAllocationsTable #scaledTickerAllocationsTable
tickerAllocationsTable = tickerAllocationsTable.fillna(0)
tickerPerformance, algoPerformance, algoTransactionCost = portfolioGeneration.calculatePerformanceForAllocations(tickerAllocationsTable, joinedData)
benchmark = factorToTrade
factorReturn = dataAck.getDailyFactorReturn(benchmark, joinedData)
factorReturn.columns = ["Factor Return (" + benchmark + ")"]
algoPerformance.columns = ["Algo Return"]
algoPerformanceRollingWeekly = algoPerformance.rolling(5, min_periods=5).apply(lambda x:empyrical.cum_returns(x)[-1]).dropna()
algoPerformanceRollingWeekly.columns = ["Weekly Rolling Performance"]
algoPerformanceRollingMonthly = algoPerformance.rolling(22, min_periods=22).apply(lambda x:empyrical.cum_returns(x)[-1]).dropna()
algoPerformanceRollingMonthly.columns = ["Monthly Rolling Performance"]
algoPerformanceRollingYearly = algoPerformance.rolling(252, min_periods=252).apply(lambda x:empyrical.cum_returns(x)[-1]).dropna()
algoPerformanceRollingYearly.columns = ["Yearly Rolling Performance"]
tickersUsed = []
for mod in modelsUsed:
tickersUsed.append(mod.targetTicker)
# for ticker in tickersUsed:
# thisFactorReturn = dataAck.getDailyFactorReturn(ticker, joinedData)
# thisFactorReturn.columns = ["Factor Return (" + ticker + ")"]
# alpha, beta = empyrical.alpha_beta(algoPerformance, thisFactorReturn)
# print(ticker, beta)
alpha, beta = empyrical.alpha_beta(algoPerformance, factorReturn)
sharpe_difference = empyrical.sharpe_ratio(algoPerformance) - empyrical.sharpe_ratio(factorReturn)
annualizedReturn = empyrical.annual_return(algoPerformance)[0]
annualizedVolatility = empyrical.annual_volatility(algoPerformance)
stability = empyrical.stability_of_timeseries(algoPerformance)
profitability = len((algoPerformance.values)[algoPerformance.values > 0])/len(algoPerformance.values)
rollingSharpe = algoPerformance.rolling(252, min_periods=252).apply(lambda x:empyrical.sharpe_ratio(x)).dropna()
rollingSharpe.columns = ["252 Day Rolling Sharpe"]
rollingSharpeError = rollingSharpe["252 Day Rolling Sharpe"].std()
rollingSharpeMinimum = np.percentile(rollingSharpe["252 Day Rolling Sharpe"].values, 1)
##AUTOMATICALLY TAKES SLIPPAGE INTO ACCOUNT
return {
"benchmark":factorToTrade,
"alpha":alpha,
"beta":abs(beta),
"sharpe difference":sharpe_difference,
"annualizedReturn":annualizedReturn,
"annualizedVolatility":annualizedVolatility,
"sharpe":empyrical.sharpe_ratio(algoPerformance),
"free return":annualizedReturn - annualizedVolatility,
"stability":stability,
"profitability":profitability,
"rollingSharpeError":rollingSharpeError,
"rollingSharpeMinimum":rollingSharpeMinimum,
"weeklyMinimum":algoPerformanceRollingWeekly.min().values[0],
"monthlyMinimum":algoPerformanceRollingMonthly.min().values[0],
"yearlyMinimum":algoPerformanceRollingYearly.min().values[0]
}, tickerAllocationsTable
def runModelHistorical(self, dataOfInterest):
##RAW PREDICTIONS ARE PREDS 0->1.0
returnStream, factorReturn, predictions, slippageAdjustedReturn, rawPredictions1 = self.obj1.runModelHistorical(
dataOfInterest)
returnStream, factorReturn, predictions, slippageAdjustedReturn, rawPredictions2 = self.obj2.runModelHistorical(
dataOfInterest)
print(rawPredictions1)
print(rawPredictions2)
#computePositionConfidence
rawPredictions1 = pd.DataFrame(rawPredictions1.apply(
lambda x: dataAck.computePosition(x), axis=1),
columns=["Predictions 1"]).dropna()
rawPredictions2 = pd.DataFrame(rawPredictions2.apply(
lambda x: dataAck.computePosition(x), axis=1),
columns=["Predictions 2"]).dropna()
print(rawPredictions1)
print(rawPredictions2)
rawPredictions = rawPredictions1.join(rawPredictions2).dropna()
print(rawPredictions)
#averagePredictions
predsTable = pd.DataFrame(
rawPredictions.apply(lambda x: self.combinePredictions(x),
axis=1,
raw=True))
rawPredictions = predsTable
##PREDICTIONS COMBINED AS 0, 0.5, 1 where
i = 1
tablesToJoin = []
while i < self.predictionDistance:
thisTable = predsTable.shift(i)
thisTable.columns = ["Predictions_" + str(i)]
tablesToJoin.append(thisTable)
i += 1
returnStream = None
factorReturn = None
predictions = None
slippageAdjustedReturn = None
predsTable = predsTable.join(tablesToJoin)
##AVERAGE...A LOT OF SUBTLETY IN STRENGTH OF PREDICTION
transformedPreds = pd.DataFrame(predsTable.apply(
lambda x: dataAck.computePosition(x), axis=1),
columns=["Predictions"]).dropna()
dailyFactorReturn = dataAck.getDailyFactorReturn(
self.targetTicker, dataOfInterest)
transformedPreds = transformedPreds.join(dailyFactorReturn).dropna()
returnStream = pd.DataFrame(transformedPreds.apply(
lambda x: x[0] * x[1], axis=1),
columns=["Algo Return"])
factorReturn = pd.DataFrame(transformedPreds[["Factor Return"]])
predictions = pd.DataFrame(transformedPreds[["Predictions"]])
estimatedSlippageLoss = portfolioGeneration.estimateTransactionCost(
predictions)
estimatedSlippageLoss.columns = returnStream.columns
slippageAdjustedReturn = (returnStream -
estimatedSlippageLoss).dropna()
return returnStream, factorReturn, predictions, slippageAdjustedReturn, rawPredictions
def getDataForPortfolio(portfolioKey):
models = portfolio.getModelsByKey(portfolio.getPortfolioModels(portfolioKey))
##DOWNLOAD REQUIRED DATA FOR TARGET TICKERS
tickersRequired = []
for mod in models:
print(mod.describe())
if mod.inputSeries.targetTicker not in tickersRequired:
tickersRequired.append(mod.inputSeries.targetTicker)
pulledData, validTickers = dataAck.downloadTickerData(tickersRequired)
joinedData = dataAck.joinDatasets([pulledData[ticker] for ticker in pulledData])
##GENERATE RETURNS FOR PORTFOLIO
portfolioAllocations = portfolio.getPortfolioAllocations(portfolioKey)
predsTable = pd.DataFrame([])
weightsTable = pd.DataFrame([])
tickerAllocationsTable = pd.DataFrame([])
scaledTickerAllocationsTable = pd.DataFrame([])
for allocation in portfolioAllocations:
colsAlgo = []
valsAlgo = []
colsAlgoWeight = []
valsAlgoWeight = []
colsTicker = []
valsTicker = []
colsTickerScaled = []
valsTickerScaled = []
for key in allocation:
if key.startswith("ticker_"):
colsTicker.append(key[len("ticker_"):])
valsTicker.append(allocation[key])
if key.startswith("scaled_ticker_"):
colsTickerScaled.append(key[len("scaled_ticker_"):])
valsTickerScaled.append(allocation[key])
if key.startswith("algo_") and not key.startswith("algo_weight_"):
colsAlgo.append(key[len("algo_"):])
valsAlgo.append(allocation[key])
if key.startswith("algo_weight_"):
colsAlgoWeight.append(key[len("algo_weight_"):])
valsAlgoWeight.append(allocation[key])
predsTable = pd.concat([predsTable, pd.DataFrame([valsAlgo], index = [allocation["predictionDay"]], columns=colsAlgo).tz_localize(None)])
weightsTable = pd.concat([weightsTable, pd.DataFrame([valsAlgoWeight], index = [allocation["predictionDay"]], columns=colsAlgoWeight).tz_localize(None)])
tickerAllocationsTable = pd.concat([tickerAllocationsTable, pd.DataFrame([valsTicker], index = [allocation["predictionDay"]], columns=colsTicker).tz_localize(None)])
scaledTickerAllocationsTable = pd.concat([scaledTickerAllocationsTable, pd.DataFrame([valsTickerScaled], index = [allocation["predictionDay"]], columns=colsTickerScaled).tz_localize(None)])
predsTable = predsTable.sort_index()
weightsTable = weightsTable.sort_index()
tickerAllocationsTable = tickerAllocationsTable.sort_index()
scaledTickerAllocationsTable = scaledTickerAllocationsTable.sort_index()
tickerPerformance = calculatePerformanceForTable(tickerAllocationsTable, tickerAllocationsTable.columns, joinedData)
algoPerformance = pd.DataFrame(tickerPerformance.apply(lambda x:sum(x), axis=1), columns=["Algo Return"])
benchmark = portfolio.getPortfolioByKey(portfolioKey)["benchmark"]
factorReturn = dataAck.getDailyFactorReturn(benchmark, joinedData)
factorReturn.columns = ["Factor Return (" + benchmark + ")"]
algoVsBenchmark = algoPerformance.join(factorReturn).dropna()
##FORM HASH TO TICKER
hashToTicker = {}
for model in models:
hashToTicker[portfolio.getModelHash(model)] = model.inputSeries.targetTicker
individualAlgoPerformance = calculatePerformanceForTable(predsTable,[hashToTicker[modelHash] for modelHash in predsTable.columns], joinedData)
return json.dumps(convertTableToJSON(empyrical.cum_returns(tickerPerformance))),\
json.dumps(convertTableToJSON(empyrical.cum_returns(algoPerformance))),\
json.dumps(convertTableToJSON(empyrical.cum_returns(algoVsBenchmark))),\
json.dumps(convertTableToJSON(empyrical.cum_returns(individualAlgoPerformance))),\
def getFundData():
historicalAllocations, realizedAllocations = getNetAllocationAcrossPortfolios(
)
if historicalAllocations is None:
return None, None
pulledData, unused_ = dataAck.downloadTickerData(
historicalAllocations.columns.values)
allocationJoinedData = dataAck.joinDatasets(
[pulledData[ticker] for ticker in pulledData])
dataToCache = []
for allocationForm in [historicalAllocations, realizedAllocations]:
performanceByTicker, fundPerformance, fundTransactionCost = portfolioGeneration.calculatePerformanceForAllocations(
allocationForm, allocationJoinedData)
if len(fundPerformance) == 0:
dataToCache.append({})
continue
##CALCULATE BETAS FOR ALL TICKERS TO FUND PERFORMANCE
tickerAlphaBetas = []
for ticker in allocationForm.columns.values:
factorReturn = dataAck.getDailyFactorReturn(
ticker, allocationJoinedData)
alpha, beta = empyrical.alpha_beta(fundPerformance, factorReturn)
tickerAlphaBetas.append({
"ticker": ticker,
"alpha": alpha * 100,
"beta": beta
})
tickerCols, tickerRows = portfolioGeneration.convertTableToJSON(
empyrical.cum_returns(performanceByTicker))
tickerAllocationsCols, tickerAllocationsRows = portfolioGeneration.convertTableToJSON(
allocationForm)
fundCols, fundRows = portfolioGeneration.convertTableToJSON(
empyrical.cum_returns(fundPerformance))
sharpe = empyrical.sharpe_ratio(fundPerformance)
annualReturn = empyrical.annual_return(fundPerformance)[0]
annualVol = empyrical.annual_volatility(fundPerformance)
commissionCols, commissionRows = portfolioGeneration.convertTableToJSON(
fundTransactionCost)
dataToCache.append({
"tickerAlphaBetas":
tickerAlphaBetas,
"tickerCols":
json.dumps(tickerCols),
"tickerRows":
json.dumps(tickerRows),
"tickerAllocationsCols":
json.dumps(tickerAllocationsCols),
"tickerAllocationsRows":
json.dumps(tickerAllocationsRows),
"fundCols":
json.dumps(fundCols),
"fundRows":
json.dumps(fundRows),
"sharpe":
sharpe,
"annualReturn":
annualReturn * 100,
"annualVol":
annualVol * 100,
"commissionCols":
json.dumps(commissionCols),
"commissionRows":
json.dumps(commissionRows)
})
historicalData = dataToCache[0]
realizedData = dataToCache[1]
##GET TODAY ALLOCATION
if realizedData != {}:
newRows = []
tARows = json.loads(realizedData["tickerAllocationsRows"])
tACols = json.loads(realizedData["tickerAllocationsCols"])
print(tARows[-1])
for i in range(len(tACols)):
newRows.append([tACols[i],
abs(tARows[-1][i + 1])]) ##i+1 because date
realizedData["todayAllocation"] = json.dumps(newRows)
print(realizedData["todayAllocation"])
return historicalData, realizedData
def runModelsChunksSkipMP(self, dataOfInterest, daysToCheck = None, earlyStop=False):
xVals, yVals, yIndex, xToday = self.generateWindows(dataOfInterest)
mpEngine = mp.get_context('fork')
with mpEngine.Manager() as manager:
returnDict = manager.dict()
identifiersToCheck = []
for i in range(len(xVals) - 44): ##44 is lag...should not overlap with any other predictions or will ruin validity of walkforward optimization
if i < 600:
##MIN TRAINING
continue
identifiersToCheck.append(str(i))
if daysToCheck is not None:
identifiersToCheck = identifiersToCheck[-daysToCheck:]
##FIRST CHECK FIRST 500 IDENTIFIERS AND THEN IF GOOD CONTINUE
identifierWindows = [identifiersToCheck[:252], identifiersToCheck[252:600], identifiersToCheck[600:900], identifiersToCheck[900:1200], identifiersToCheck[1200:]] ##EXACTLY TWO YEARS
if earlyStop == False:
identifierWindows = [identifiersToCheck]
returnStream = None
factorReturn = None
predictions = None
slippageAdjustedReturn = None
rawPredictions = None
shortSeen = 0 if earlyStop == True else -1
for clippedIdentifiers in identifierWindows:
splitIdentifiers = np.array_split(np.array(clippedIdentifiers), 4)
runningP = []
k = 0
for identifiers in splitIdentifiers:
p = mpEngine.Process(target=CurvePredictor.runDayChunking, args=(self, xVals, yVals, identifiers, returnDict,k))
p.start()
runningP.append(p)
k += 1
while len(runningP) > 0:
newP = []
for p in runningP:
if p.is_alive() == True:
newP.append(p)
else:
p.join()
runningP = newP
preds = []
actuals = []
days = []
for i in clippedIdentifiers:
preds.append(returnDict[i])
actuals.append(yVals[int(i) + 44])
days.append(yIndex[int(i) + 44])
##CREATE ACCURATE BLENDING ACROSS DAYS
predsTable = pd.DataFrame(preds, index=days, columns=["Predictions"])
i = 1
tablesToJoin = []
while i < self.predictionDistance:
thisTable = predsTable.shift(i)
thisTable.columns = ["Predictions_" + str(i)]
tablesToJoin.append(thisTable)
i += 1
predsTable = predsTable.join(tablesToJoin)
transformedPreds = pd.DataFrame(predsTable.apply(lambda x:dataAck.computePosition(x), axis=1), columns=["Predictions"]).dropna()
dailyFactorReturn = dataAck.getDailyFactorReturn(self.targetTicker, dataOfInterest)
transformedPreds = transformedPreds.join(dailyFactorReturn).dropna()
returnStream = pd.DataFrame(transformedPreds.apply(lambda x:x[0] * x[1], axis=1), columns=["Algo Return"]) if returnStream is None else pd.concat([returnStream, pd.DataFrame(transformedPreds.apply(lambda x:x[0] * x[1], axis=1), columns=["Algo Return"])])
factorReturn = pd.DataFrame(transformedPreds[["Factor Return"]]) if factorReturn is None else pd.concat([factorReturn, pd.DataFrame(transformedPreds[["Factor Return"]])])
predictions = pd.DataFrame(transformedPreds[["Predictions"]]) if predictions is None else pd.concat([predictions, pd.DataFrame(transformedPreds[["Predictions"]])])
rawPredictions = pd.DataFrame(preds, index=days, columns=["Predictions"]) if rawPredictions is None else pd.concat([rawPredictions, pd.DataFrame(preds, index=days, columns=["Predictions"])])
alpha, beta = empyrical.alpha_beta(returnStream, factorReturn)
activity = np.count_nonzero(returnStream)/float(len(returnStream))
rawBeta = abs(empyrical.alpha_beta(returnStream.apply(lambda x:dataAck.applyBinary(x), axis=0), factorReturn.apply(lambda x:dataAck.applyBinary(x), axis=0))[1])
shortSharpe = empyrical.sharpe_ratio(returnStream)
activity = np.count_nonzero(returnStream)/float(len(returnStream))
algoAnnualReturn = empyrical.annual_return(returnStream.values)[0]
algoVol = empyrical.annual_volatility(returnStream.values)
factorAnnualReturn = empyrical.annual_return(factorReturn.values)[0]
factorVol = empyrical.annual_volatility(factorReturn.values)
treynor = ((empyrical.annual_return(returnStream.values)[0] - empyrical.annual_return(factorReturn.values)[0]) \
/ abs(empyrical.beta(returnStream, factorReturn)))
sharpeDiff = empyrical.sharpe_ratio(returnStream) - empyrical.sharpe_ratio(factorReturn)
relativeSharpe = sharpeDiff / empyrical.sharpe_ratio(factorReturn) * (empyrical.sharpe_ratio(factorReturn)/abs(empyrical.sharpe_ratio(factorReturn)))
stability = empyrical.stability_of_timeseries(returnStream)
##CALCULATE SHARPE WITH SLIPPAGE
estimatedSlippageLoss = portfolioGeneration.estimateTransactionCost(predictions)
estimatedSlippageLoss.columns = returnStream.columns
slippageAdjustedReturn = (returnStream - estimatedSlippageLoss).dropna()
slippageSharpe = empyrical.sharpe_ratio(slippageAdjustedReturn)
sharpeDiffSlippage = empyrical.sharpe_ratio(slippageAdjustedReturn) - empyrical.sharpe_ratio(factorReturn)
relativeSharpeSlippage = sharpeDiffSlippage / empyrical.sharpe_ratio(factorReturn) * (empyrical.sharpe_ratio(factorReturn)/abs(empyrical.sharpe_ratio(factorReturn)))
profitability = len((returnStream.values)[returnStream.values > 0])/len(returnStream.values)
rollingProfitability = returnStream.rolling(45, min_periods=45).apply(lambda x:len((x)[x > 0])/len(x)).dropna().values
minRollingProfitability = np.percentile(rollingProfitability, 1)
twentyFifthPercentileRollingProfitablity = np.percentile(rollingProfitability, 25)
if np.isnan(shortSharpe) == True:
return None, {"sharpe":shortSharpe}, None, None, None
elif (profitability < 0.4 or activity < 0.3 or abs(rawBeta) > 0.4 or stability < 0.3) and shortSeen == 0:
return None, {
"sharpe":shortSharpe, ##OVERLOADED IN FAIL
"activity":activity,
"factorSharpe":empyrical.sharpe_ratio(factorReturn),
"sharpeSlippage":slippageSharpe,
"beta":abs(beta),
"alpha":alpha,
"activity":activity,
"treynor":treynor,
"period":"first 252 days",
"algoReturn":algoAnnualReturn,
"algoVol":algoVol,
"factorReturn":factorAnnualReturn,
"factorVol":factorVol,
"sharpeDiff":sharpeDiff,
"relativeSharpe":relativeSharpe,
"sharpeDiffSlippage":sharpeDiffSlippage,
"relativeSharpeSlippage":relativeSharpeSlippage,
"rawBeta":rawBeta,
"minRollingProfitability":minRollingProfitability,
"stability":stability,
"twentyFifthPercentileRollingProfitablity":twentyFifthPercentileRollingProfitablity,
"profitability":profitability
}, None, None, None
elif abs(rawBeta) > 0.33 or activity < 0.3 or stability < 0.4 or twentyFifthPercentileRollingProfitablity < 0.41 \
or minRollingProfitability < 0.3 or profitability < 0.46:
periodName = "first 600 days"
if shortSeen == 2:
periodName = "first 900 days"
elif shortSeen == 3:
periodName = "first 1200 days"
return None, {
"sharpe":shortSharpe, ##OVERLOADED IN FAIL
"activity":activity,
"factorSharpe":empyrical.sharpe_ratio(factorReturn),
"sharpeSlippage":slippageSharpe,
"alpha":alpha,
"beta":abs(beta),
"activity":activity,
"treynor":treynor,
"period":periodName,
"algoReturn":algoAnnualReturn,
"algoVol":algoVol,
"factorReturn":factorAnnualReturn,
"factorVol":factorVol,
"minRollingProfitability":minRollingProfitability,
"sharpeDiff":sharpeDiff,
"relativeSharpe":relativeSharpe,
"sharpeDiffSlippage":sharpeDiffSlippage,
"relativeSharpeSlippage":relativeSharpeSlippage,
"rawBeta":rawBeta,
"stability":stability,
"twentyFifthPercentileRollingProfitablity":twentyFifthPercentileRollingProfitablity,
"profitability":profitability
}, None, None, None
elif shortSeen < 4:
print("CONTINUING", "SHARPE:", shortSharpe, "SHARPE DIFF:", sharpeDiff, "RAW BETA:", rawBeta, "TREYNOR:", treynor)
shortSeen += 1
return returnStream, factorReturn, predictions, slippageAdjustedReturn, rawPredictions