class BlackLittermanOptimizationPortfolioConstructionModel(PortfolioConstructionModel):
def __init__(self,
lookback = 1,
period = 63,
resolution = Resolution.Daily,
risk_free_rate = 0,
delta = 2.5,
tau = 0.05,
optimizer = None):
"""Initialize the model
Args:
lookback(int): Historical return lookback period
period(int): The time interval of history price to calculate the weight
resolution: The resolution of the history price
risk_free_rate(float): The risk free rate
delta(float): The risk aversion coeffficient of the market portfolio
tau(float): The model parameter indicating the uncertainty of the CAPM prior"""
self.lookback = lookback
self.period = period
self.resolution = resolution
self.risk_free_rate = risk_free_rate
self.delta = delta
self.tau = tau
self.optimizer = MaximumSharpeRatioPortfolioOptimizer(risk_free_rate = risk_free_rate) if optimizer is None else optimizer
self.removedSymbols = []
self.symbolDataBySymbol = {}
self.insightCollection = InsightCollection()
self.nextExpiryTime = UTCMIN
self.rebalancingTime = UTCMIN
self.rebalancingPeriod = Extensions.ToTimeSpan(resolution)
def CreateTargets(self, algorithm, insights):
"""
Create portfolio targets from the specified insights
Args:
algorithm: The algorithm instance
insights: The insights to create portfolio targets from
Returns:
An enumerable of portfolio targets to be sent to the execution model
"""
targets = []
if (algorithm.UtcTime <= self.nextExpiryTime and
algorithm.UtcTime <= self.rebalancingTime and
len(insights) == 0 and
self.removedSymbols is None):
return targets
self.insightCollection.AddRange(insights)
# Create flatten target for each security that was removed from the universe
if self.removedSymbols is not None:
universeDeselectionTargets = [ PortfolioTarget(symbol, 0) for symbol in self.removedSymbols ]
targets.extend(universeDeselectionTargets)
self.removedSymbols = None
# Get insight that haven't expired of each symbol that is still in the universe
activeInsights = self.insightCollection.GetActiveInsights(algorithm.UtcTime)
# Get the last generated active insight for each symbol
lastActiveInsights = []
for sourceModel, f in groupby(sorted(activeInsights, key = lambda ff: ff.SourceModel), lambda fff: fff.SourceModel):
for symbol, g in groupby(sorted(list(f), key = lambda gg: gg.Symbol), lambda ggg: ggg.Symbol):
lastActiveInsights.append(sorted(g, key = lambda x: x.GeneratedTimeUtc)[-1])
# Get view vectors
P, Q = self.get_views(lastActiveInsights)
if P is not None:
returns = dict()
# Updates the BlackLittermanSymbolData with insights
# Create a dictionary keyed by the symbols in the insights with an pandas.Series as value to create a data frame
for insight in lastActiveInsights:
symbol = insight.Symbol
symbolData = self.symbolDataBySymbol.get(symbol, self.BlackLittermanSymbolData(insight.Symbol, self.lookback, self.period))
if insight.Magnitude is None:
algorithm.SetRunTimeError(ArgumentNullExceptionArgumentNullException('BlackLittermanOptimizationPortfolioConstructionModel does not accept \'None\' as Insight.Magnitude. Please make sure your Alpha Model is generating Insights with the Magnitude property set.'))
symbolData.Add(algorithm.Time, insight.Magnitude)
returns[symbol] = symbolData.Return
returns = pd.DataFrame(returns)
# Calculate prior estimate of the mean and covariance
Pi, Sigma = self.get_equilibrium_return(returns)
# Calculate posterior estimate of the mean and covariance
Pi, Sigma = self.apply_blacklitterman_master_formula(Pi, Sigma, P, Q)
# Create portfolio targets from the specified insights
weights = self.optimizer.Optimize(returns, Pi, Sigma)
weights = pd.Series(weights, index = Sigma.columns)
for symbol, weight in weights.items():
target = PortfolioTarget.Percent(algorithm, symbol, weight)
if target is not None:
targets.append(target)
# Get expired insights and create flatten targets for each symbol
expiredInsights = self.insightCollection.RemoveExpiredInsights(algorithm.UtcTime)
expiredTargets = []
for symbol, f in groupby(expiredInsights, lambda x: x.Symbol):
if not self.insightCollection.HasActiveInsights(symbol, algorithm.UtcTime):
expiredTargets.append(PortfolioTarget(symbol, 0))
continue
targets.extend(expiredTargets)
self.nextExpiryTime = self.insightCollection.GetNextExpiryTime()
if self.nextExpiryTime is None:
self.nextExpiryTime = UTCMIN
self.rebalancingTime = algorithm.UtcTime + self.rebalancingPeriod
return targets
def OnSecuritiesChanged(self, algorithm, changes):
'''Event fired each time the we add/remove securities from the data feed
Args:
algorithm: The algorithm instance that experienced the change in securities
changes: The security additions and removals from the algorithm'''
# Get removed symbol and invalidate them in the insight collection
self.removedSymbols = [x.Symbol for x in changes.RemovedSecurities]
self.insightCollection.Clear(self.removedSymbols)
for symbol in self.removedSymbols:
symbolData = self.symbolDataBySymbol.pop(symbol, None)
if symbolData is not None:
symbolData.Reset()
# initialize data for added securities
addedSymbols = [ x.Symbol for x in changes.AddedSecurities ]
history = algorithm.History(addedSymbols, self.lookback * self.period, self.resolution)
for symbol in addedSymbols:
symbolData = self.BlackLittermanSymbolData(symbol, self.lookback, self.period)
if not history.empty:
ticker = SymbolCache.GetTicker(symbol)
if ticker not in history.index.levels[0]:
Log.Trace(f'BlackLittermanOptimizationPortfolioConstructionModel.OnSecuritiesChanged: {ticker} not found in history data frame.')
continue
symbolData.WarmUpIndicators(history.loc[ticker])
self.symbolDataBySymbol[symbol] = symbolData
def apply_blacklitterman_master_formula(self, Pi, Sigma, P, Q):
'''Apply Black-Litterman master formula
http://www.blacklitterman.org/cookbook.html
Args:
Pi: Prior/Posterior mean array
Sigma: Prior/Posterior covariance matrix
P: A matrix that identifies the assets involved in the views (size: K x N)
Q: A view vector (size: K x 1)'''
ts = self.tau * Sigma
# Create the diagonal Sigma matrix of error terms from the expressed views
omega = np.dot(np.dot(P, ts), P.T) * np.eye(Q.shape[0])
if np.linalg.det(omega) == 0:
return Pi, Sigma
A = np.dot(np.dot(ts, P.T), inv(np.dot(np.dot(P, ts), P.T) + omega))
Pi = np.squeeze(np.asarray((
np.expand_dims(Pi, axis=0).T +
np.dot(A, (Q - np.expand_dims(np.dot(P, Pi.T), axis=1))))
))
M = ts - np.dot(np.dot(A, P), ts)
Sigma = (Sigma + M) * self.delta
return Pi, Sigma
def get_equilibrium_return(self, returns):
'''Calculate equilibrium returns and covariance
Args:
returns: Matrix of returns where each column represents a security and each row returns for the given date/time (size: K x N)
Returns:
equilibrium_return: Array of double of equilibrium returns
cov: Multi-dimensional array of double with the portfolio covariance of returns (size: K x K)'''
size = len(returns.columns)
# equal weighting scheme
W = np.array([1/size]*size)
# the covariance matrix of excess returns (N x N matrix)
cov = returns.cov()*252
# annualized return
annual_return = np.sum(((1 + returns.mean())**252 -1) * W)
# annualized variance of return
annual_variance = dot(W.T, dot(cov, W))
# the risk aversion coefficient
risk_aversion = (annual_return - self.risk_free_rate ) / annual_variance
# the implied excess equilibrium return Vector (N x 1 column vector)
equilibrium_return = dot(dot(risk_aversion, cov), W)
return equilibrium_return, cov
def get_views(self, insights):
'''Generate views from multiple alpha models
Args
insights: Array of insight that represent the investors' views
Returns
P: A matrix that identifies the assets involved in the views (size: K x N)
Q: A view vector (size: K x 1)'''
try:
P = {}
Q = {}
for model, group in groupby(insights, lambda x: x.SourceModel):
group = list(group)
up_insights_sum = 0.0
dn_insights_sum = 0.0
for insight in group:
if insight.Direction == InsightDirection.Up:
up_insights_sum = up_insights_sum + np.abs(insight.Magnitude)
if insight.Direction == InsightDirection.Down:
dn_insights_sum = dn_insights_sum + np.abs(insight.Magnitude)
q = up_insights_sum if up_insights_sum > dn_insights_sum else dn_insights_sum
if q == 0:
continue
Q[model] = q
# generate the link matrix of views: P
P[model] = dict()
for insight in group:
value = insight.Direction * np.abs(insight.Magnitude)
P[model][insight.Symbol] = value / q
# Add zero for other symbols that are listed but active insight
for symbol in self.symbolDataBySymbol.keys():
if symbol not in P[model]:
P[model][symbol] = 0
Q = np.array([[x] for x in Q.values()])
if len(Q) > 0:
P = np.array([list(x.values()) for x in P.values()])
return P, Q
except:
pass
return None, None
class BlackLittermanSymbolData:
'''Contains data specific to a symbol required by this model'''
def __init__(self, symbol, lookback, period):
self.symbol = symbol
self.roc = RateOfChange(f'{symbol}.ROC({lookback})', lookback)
self.roc.Updated += self.OnRateOfChangeUpdated
self.window = RollingWindow[IndicatorDataPoint](period)
def Reset(self):
self.roc.Updated -= self.OnRateOfChangeUpdated
self.roc.Reset()
self.window.Reset()
def WarmUpIndicators(self, history):
for tuple in history.itertuples():
self.roc.Update(tuple.Index, tuple.close)
def OnRateOfChangeUpdated(self, roc, value):
if roc.IsReady:
self.window.Add(value)
def Add(self, time, value):
item = IndicatorDataPoint(self.symbol, time, value)
self.window.Add(item)
@property
def Return(self):
return pd.Series(
data = [float(x.Value) for x in self.window],
index = [x.EndTime for x in self.window])
@property
def IsReady(self):
return self.window.IsReady
def __str__(self, **kwargs):
return '{}: {:.2%}'.format(self.roc.Name, (1 + self.window[0])**252 - 1)
class BlackLittermanOptimizationPortfolioConstructionModel(
PortfolioConstructionModel):
def __init__(self,
rebalance=Resolution.Daily,
portfolioBias=PortfolioBias.LongShort,
lookback=1,
period=63,
resolution=Resolution.Daily,
risk_free_rate=0,
delta=2.5,
tau=0.05,
optimizer=None):
"""Initialize the model
Args:
rebalance: Rebalancing parameter. If it is a timedelta, date rules or Resolution, it will be converted into a function.
If None will be ignored.
The function returns the next expected rebalance time for a given algorithm UTC DateTime.
The function returns null if unknown, in which case the function will be called again in the
next loop. Returning current time will trigger rebalance.
portfolioBias: Specifies the bias of the portfolio (Short, Long/Short, Long)
lookback(int): Historical return lookback period
period(int): The time interval of history price to calculate the weight
resolution: The resolution of the history price
risk_free_rate(float): The risk free rate
delta(float): The risk aversion coeffficient of the market portfolio
tau(float): The model parameter indicating the uncertainty of the CAPM prior"""
super().__init__()
self.lookback = lookback
self.period = period
self.resolution = resolution
self.risk_free_rate = risk_free_rate
self.delta = delta
self.tau = tau
self.portfolioBias = portfolioBias
lower = 0 if portfolioBias == PortfolioBias.Long else -1
upper = 0 if portfolioBias == PortfolioBias.Short else 1
self.optimizer = MaximumSharpeRatioPortfolioOptimizer(
lower, upper, risk_free_rate) if optimizer is None else optimizer
self.sign = lambda x: -1 if x < 0 else (1 if x > 0 else 0)
self.symbolDataBySymbol = {}
# If the argument is an instance of Resolution or Timedelta
# Redefine rebalancingFunc
rebalancingFunc = rebalance
if isinstance(rebalance, int):
rebalance = Extensions.ToTimeSpan(rebalance)
if isinstance(rebalance, timedelta):
rebalancingFunc = lambda dt: dt + rebalance
if rebalancingFunc:
self.SetRebalancingFunc(rebalancingFunc)
def ShouldCreateTargetForInsight(self, insight):
return len(
PortfolioConstructionModel.FilterInvalidInsightMagnitude(
self.Algorithm, [insight])) != 0
def DetermineTargetPercent(self, lastActiveInsights):
targets = {}
# Get view vectors
P, Q = self.get_views(lastActiveInsights)
if P is not None:
returns = dict()
# Updates the BlackLittermanSymbolData with insights
# Create a dictionary keyed by the symbols in the insights with an pandas.Series as value to create a data frame
for insight in lastActiveInsights:
symbol = insight.Symbol
symbolData = self.symbolDataBySymbol.get(
symbol,
self.BlackLittermanSymbolData(symbol, self.lookback,
self.period))
if insight.Magnitude is None:
self.Algorithm.SetRunTimeError(
ArgumentNullException(
'BlackLittermanOptimizationPortfolioConstructionModel does not accept \'None\' as Insight.Magnitude. Please make sure your Alpha Model is generating Insights with the Magnitude property set.'
))
return targets
symbolData.Add(insight.GeneratedTimeUtc, insight.Magnitude)
returns[symbol] = symbolData.Return
returns = pd.DataFrame(returns)
# Calculate prior estimate of the mean and covariance
Pi, Sigma = self.get_equilibrium_return(returns)
# Calculate posterior estimate of the mean and covariance
Pi, Sigma = self.apply_blacklitterman_master_formula(
Pi, Sigma, P, Q)
# Create portfolio targets from the specified insights
weights = self.optimizer.Optimize(returns, Pi, Sigma)
weights = pd.Series(weights, index=Sigma.columns)
for symbol, weight in weights.items():
for insight in lastActiveInsights:
if str(insight.Symbol) == str(symbol):
# don't trust the optimizer
if self.portfolioBias != PortfolioBias.LongShort and self.sign(
weight) != self.portfolioBias:
weight = 0
targets[insight] = weight
break
return targets
def GetTargetInsights(self):
# Get insight that haven't expired of each symbol that is still in the universe
activeInsights = self.InsightCollection.GetActiveInsights(
self.Algorithm.UtcTime)
# Get the last generated active insight for each symbol
lastActiveInsights = []
for sourceModel, f in groupby(
sorted(activeInsights, key=lambda ff: ff.SourceModel),
lambda fff: fff.SourceModel):
for symbol, g in groupby(sorted(list(f), key=lambda gg: gg.Symbol),
lambda ggg: ggg.Symbol):
lastActiveInsights.append(
sorted(g, key=lambda x: x.GeneratedTimeUtc)[-1])
return lastActiveInsights
def OnSecuritiesChanged(self, algorithm, changes):
'''Event fired each time the we add/remove securities from the data feed
Args:
algorithm: The algorithm instance that experienced the change in securities
changes: The security additions and removals from the algorithm'''
# Get removed symbol and invalidate them in the insight collection
super().OnSecuritiesChanged(algorithm, changes)
for security in changes.RemovedSecurities:
symbol = security.Symbol
symbolData = self.symbolDataBySymbol.pop(symbol, None)
if symbolData is not None:
symbolData.Reset()
# initialize data for added securities
addedSymbols = {
x.Symbol: x.Exchange.TimeZone
for x in changes.AddedSecurities
}
history = algorithm.History(list(addedSymbols.keys()),
self.lookback * self.period,
self.resolution)
if history.empty:
return
history = history.close.unstack(0)
symbols = history.columns
for symbol, timezone in addedSymbols.items():
if str(symbol) not in symbols:
continue
symbolData = self.symbolDataBySymbol.get(
symbol,
self.BlackLittermanSymbolData(symbol, self.lookback,
self.period))
for time, close in history[symbol].items():
utcTime = Extensions.ConvertToUtc(time, timezone)
symbolData.Update(utcTime, close)
self.symbolDataBySymbol[symbol] = symbolData
def apply_blacklitterman_master_formula(self, Pi, Sigma, P, Q):
'''Apply Black-Litterman master formula
http://www.blacklitterman.org/cookbook.html
Args:
Pi: Prior/Posterior mean array
Sigma: Prior/Posterior covariance matrix
P: A matrix that identifies the assets involved in the views (size: K x N)
Q: A view vector (size: K x 1)'''
ts = self.tau * Sigma
# Create the diagonal Sigma matrix of error terms from the expressed views
omega = np.dot(np.dot(P, ts), P.T) * np.eye(Q.shape[0])
if np.linalg.det(omega) == 0:
return Pi, Sigma
A = np.dot(np.dot(ts, P.T), inv(np.dot(np.dot(P, ts), P.T) + omega))
Pi = np.squeeze(
np.asarray(
(np.expand_dims(Pi, axis=0).T +
np.dot(A, (Q - np.expand_dims(np.dot(P, Pi.T), axis=1))))))
M = ts - np.dot(np.dot(A, P), ts)
Sigma = (Sigma + M) * self.delta
return Pi, Sigma
def get_equilibrium_return(self, returns):
'''Calculate equilibrium returns and covariance
Args:
returns: Matrix of returns where each column represents a security and each row returns for the given date/time (size: K x N)
Returns:
equilibrium_return: Array of double of equilibrium returns
cov: Multi-dimensional array of double with the portfolio covariance of returns (size: K x K)'''
size = len(returns.columns)
# equal weighting scheme
W = np.array([1 / size] * size)
# the covariance matrix of excess returns (N x N matrix)
cov = returns.cov() * 252
# annualized return
annual_return = np.sum(((1 + returns.mean())**252 - 1) * W)
# annualized variance of return
annual_variance = dot(W.T, dot(cov, W))
# the risk aversion coefficient
risk_aversion = (annual_return - self.risk_free_rate) / annual_variance
# the implied excess equilibrium return Vector (N x 1 column vector)
equilibrium_return = dot(dot(risk_aversion, cov), W)
return equilibrium_return, cov
def get_views(self, insights):
'''Generate views from multiple alpha models
Args
insights: Array of insight that represent the investors' views
Returns
P: A matrix that identifies the assets involved in the views (size: K x N)
Q: A view vector (size: K x 1)'''
try:
P = {}
Q = {}
for model, group in groupby(insights, lambda x: x.SourceModel):
group = list(group)
up_insights_sum = 0.0
dn_insights_sum = 0.0
for insight in group:
if insight.Direction == InsightDirection.Up:
up_insights_sum = up_insights_sum + np.abs(
insight.Magnitude)
if insight.Direction == InsightDirection.Down:
dn_insights_sum = dn_insights_sum + np.abs(
insight.Magnitude)
q = up_insights_sum if up_insights_sum > dn_insights_sum else dn_insights_sum
if q == 0:
continue
Q[model] = q
# generate the link matrix of views: P
P[model] = dict()
for insight in group:
value = insight.Direction * np.abs(insight.Magnitude)
P[model][insight.Symbol] = value / q
# Add zero for other symbols that are listed but active insight
for symbol in self.symbolDataBySymbol.keys():
if symbol not in P[model]:
P[model][symbol] = 0
Q = np.array([[x] for x in Q.values()])
if len(Q) > 0:
P = np.array([list(x.values()) for x in P.values()])
return P, Q
except:
pass
return None, None
class BlackLittermanSymbolData:
'''Contains data specific to a symbol required by this model'''
def __init__(self, symbol, lookback, period):
self.symbol = symbol
self.roc = RateOfChange(f'{symbol}.ROC({lookback})', lookback)
self.roc.Updated += self.OnRateOfChangeUpdated
self.window = RollingWindow[IndicatorDataPoint](period)
def Reset(self):
self.roc.Updated -= self.OnRateOfChangeUpdated
self.roc.Reset()
self.window.Reset()
def Update(self, utcTime, close):
self.roc.Update(utcTime, close)
def OnRateOfChangeUpdated(self, roc, value):
if roc.IsReady:
self.window.Add(value)
def Add(self, time, value):
if self.window.Samples > 0 and self.window[0].EndTime == time:
return
item = IndicatorDataPoint(self.symbol, time, value)
self.window.Add(item)
@property
def Return(self):
return pd.Series(data=[x.Value for x in self.window],
index=[x.EndTime for x in self.window])
@property
def IsReady(self):
return self.window.IsReady
def __str__(self, **kwargs):
return f'{self.roc.Name}: {(1 + self.window[0])**252 - 1:.2%}'