def get_mu_sigma(prices, returns_model='mean_historical_return', risk_model='ledoit_wolf',
frequency=252, span=500):
"""Get mu (returns) and sigma (asset risk) given a expected returns model and risk model
prices (pd.DataFrame) – adjusted closing prices of the asset,
each row is a date and each column is a ticker/id.
returns_model (string, optional) - Model for estimating expected returns of assets,
either 'mean_historical_return' or 'ema_historical_return' (default: mean_historical_return)
risk_model (string, optional) - Risk model to quantify risk: sample_cov, ledoit_wolf,
defaults to ledoit_wolf, as recommended by Quantopian in their lecture series on quantitative finance.
frequency (int, optional) – number of time periods in a year, defaults to 252 (the number of trading days in a year)
span (int, optional) – Applicable only for 'ema_historical_return' expected returns.
The time-span for the EMA, defaults to 500-day EMA)
"""
CHOICES_EXPECTED_RETURNS = {
'mean_historical_return': expected_returns.mean_historical_return(prices, frequency),
'ema_historical_return': expected_returns.ema_historical_return(prices, frequency, span)
}
CHOICES_RISK_MODEL = {
'sample_cov': risk_models.sample_cov(prices),
'ledoit_wolf': risk_models.CovarianceShrinkage(prices).ledoit_wolf()
}
mu = CHOICES_EXPECTED_RETURNS.get(returns_model.lower(), None)
S = CHOICES_RISK_MODEL.get(risk_model.lower(), None)
if mu is None:
raise Exception('Expected returns model %s is not supported. Only mean_historical_return and ema_historical_return are supported currently.' % risk_model)
if S is None:
raise Exception('Risk model %s is not supported. Only sample_cov and ledoit_wolf are supported currently.' % risk_model)
return mu, S
def test_correlation_plot():
plt.figure()
df = get_data()
S = risk_models.CovarianceShrinkage(df).ledoit_wolf()
ax = plotting.plot_covariance(S, showfig=False)
assert len(ax.findobj()) == 256
plt.clf()
ax = plotting.plot_covariance(S, plot_correlation=True, showfig=False)
assert len(ax.findobj()) == 256
plt.clf()
ax = plotting.plot_covariance(S, show_tickers=False, showfig=False)
assert len(ax.findobj()) == 136
plt.clf()
ax = plotting.plot_covariance(S,
plot_correlation=True,
show_tickers=False,
showfig=False)
assert len(ax.findobj()) == 136
plt.clf()
plot_filename = "tests/plot.png"
ax = plotting.plot_covariance(S, filename=plot_filename, showfig=False)
assert len(ax.findobj()) == 256
assert os.path.exists(plot_filename)
assert os.path.getsize(plot_filename) > 0
os.remove(plot_filename)
plt.clf()
plt.close()
def allocate(df):
tickers = [df.columns[k] for k in range(df.shape[1])]
ohlc = yf.download(tickers, start="2010-01-01", end="2020-01-01")
prices = ohlc["Adj Close"]
market_prices = yf.download("^BVSP", start="2010-01-01",
end="2020-01-01")["Adj Close"]
mcaps = {}
for t in tickers:
stock = yf.Ticker(t)
mcaps[t] = stock.info["marketCap"]
S = risk_models.CovarianceShrinkage(prices).ledoit_wolf()
delta = black_litterman.market_implied_risk_aversion(market_prices)
market_prior = black_litterman.market_implied_prior_returns(
mcaps, delta, S)
bl = BlackLittermanModel(S,
pi="market",
market_caps=mcaps,
risk_aversion=delta,
absolute_views=df.to_dict('records')[0])
ret_bl = bl.bl_returns()
S_bl = bl.bl_cov()
ef = EfficientFrontier(ret_bl, S_bl)
ef.add_objective(objective_functions.L2_reg)
ef.max_sharpe()
weights = ef.clean_weights()
return weights
def min_var(my_portfolio, perf=True) -> list:
ohlc = yf.download(
my_portfolio.portfolio,
start=my_portfolio.start_date,
end=my_portfolio.end_date,
progress=False,
)
prices = ohlc["Adj Close"].dropna(how="all")
prices = prices.filter(my_portfolio.portfolio)
mu = expected_returns.capm_return(prices)
S = risk_models.CovarianceShrinkage(prices).ledoit_wolf()
ef = EfficientFrontier(mu, S)
ef.add_objective(objective_functions.L2_reg,
gamma=my_portfolio.diversification)
if my_portfolio.min_weights is not None:
ef.add_constraint(lambda x: x >= my_portfolio.min_weights)
if my_portfolio.max_weights is not None:
ef.add_constraint(lambda x: x <= my_portfolio.max_weights)
ef.min_volatility()
weights = ef.clean_weights()
wts = weights.items()
result = []
for val in wts:
a, b = map(list, zip(*[val]))
result.append(b)
if perf is True:
ef.portfolio_performance(verbose=True)
return flatten(result)
def test_correlation_plot():
plt.figure()
df = get_data()
S = risk_models.CovarianceShrinkage(df).ledoit_wolf()
ax = plotting.plot_covariance(S, showfig=False)
assert len(ax.findobj()) == 256
plt.clf()
ax = plotting.plot_covariance(S, plot_correlation=True, showfig=False)
assert len(ax.findobj()) == 256
plt.clf()
ax = plotting.plot_covariance(S, show_tickers=False, showfig=False)
assert len(ax.findobj()) == 136
plt.clf()
ax = plotting.plot_covariance(S,
plot_correlation=True,
show_tickers=False,
showfig=False)
assert len(ax.findobj()) == 136
plt.clf()
temp_folder = tempfile.TemporaryDirectory()
temp_folder_path = temp_folder.name
plot_filename = os.path.join(temp_folder_path, "plot.png")
ax = plotting.plot_covariance(S, filename=plot_filename, showfig=False)
assert len(ax.findobj()) == 256
assert os.path.exists(plot_filename)
assert os.path.getsize(plot_filename) > 0
temp_folder.cleanup()
plt.clf()
plt.close()
def get_corr(org_data):
cov = risk_models.CovarianceShrinkage(org_data).ledoit_wolf()
var = np.eye(cov.shape[0]) * cov
std = np.power(var, 0.5)
I = np.linalg.inv(std)
corr = I.dot(cov).dot(I)
return corr
def test_correlation_plot():
df = get_data()
S = risk_models.CovarianceShrinkage(df).ledoit_wolf()
ax = Plotting.plot_covariance(S, showfig=False)
assert len(ax.findobj()) == 256
ax = Plotting.plot_covariance(S, show_tickers=False, showfig=False)
assert len(ax.findobj()) == 136
def test_shrunk_covariance_frequency():
df = get_data()
cs = risk_models.CovarianceShrinkage(df, frequency=52)
# if delta = 0, no shrinkage occurs
shrunk_cov = cs.shrunk_covariance(0)
S = risk_models.sample_cov(df, frequency=52)
np.testing.assert_array_almost_equal(shrunk_cov.values, S)
def test_oracle_approximating():
df = get_data()
cs = risk_models.CovarianceShrinkage(df)
shrunk_cov = cs.oracle_approximating()
assert 0 < cs.delta < 1
assert shrunk_cov.shape == (20, 20)
assert list(shrunk_cov.index) == list(df.columns)
assert list(shrunk_cov.columns) == list(df.columns)
assert not shrunk_cov.isnull().any().any()
def test_ledoit_wolf():
df = get_data()
cs = risk_models.CovarianceShrinkage(df)
shrunk_cov = cs.ledoit_wolf()
assert 0 < cs.delta < 1
assert shrunk_cov.shape == (20, 20)
assert list(shrunk_cov.index) == list(df.columns)
assert list(shrunk_cov.columns) == list(df.columns)
assert not shrunk_cov.isnull().any().any()
def test_shrunk_covariance():
df = get_data()
cs = risk_models.CovarianceShrinkage(df)
shrunk_cov = cs.shrunk_covariance(0.2)
assert cs.delta == 0.2
assert shrunk_cov.shape == (20, 20)
assert list(shrunk_cov.index) == list(df.columns)
assert list(shrunk_cov.columns) == list(df.columns)
assert not shrunk_cov.isnull().any().any()
assert risk_models._is_positive_semidefinite(shrunk_cov)
with pytest.warns(RuntimeWarning) as w:
cs_numpy = risk_models.CovarianceShrinkage(df.to_numpy())
assert len(w) == 1
assert str(w[0].message) == "data is not in a dataframe"
shrunk_cov_numpy = cs_numpy.shrunk_covariance(0.2)
assert isinstance(shrunk_cov_numpy, pd.DataFrame)
np.testing.assert_equal(shrunk_cov_numpy.to_numpy(),
shrunk_cov.to_numpy())
def test_shrunk_covariance():
df = get_data()
cs = risk_models.CovarianceShrinkage(df)
shrunk_cov = cs.shrunk_covariance(0.2)
assert cs.delta == 0.2
assert shrunk_cov.shape == (20, 20)
assert list(shrunk_cov.index) == list(df.columns)
assert list(shrunk_cov.columns) == list(df.columns)
assert not shrunk_cov.isnull().any().any()
def test_bl_cov_default():
df = get_data()
cov_matrix = risk_models.CovarianceShrinkage(df).ledoit_wolf()
viewdict = {"AAPL": 0.20, "BBY": -0.30, "BAC": 0, "SBUX": -0.2, "T": 0.131321}
bl = BlackLittermanModel(cov_matrix, absolute_views=viewdict)
S = bl.bl_cov()
assert S.shape == (20, 20)
assert S.index.equals(df.columns)
assert S.index.equals(S.columns)
assert S.notnull().all().all()
def test_ledoit_wolf_constant_correlation():
df = get_data()
cs = risk_models.CovarianceShrinkage(df)
shrunk_cov = cs.ledoit_wolf(shrinkage_target="constant_correlation")
assert 0 < cs.delta < 1
assert shrunk_cov.shape == (20, 20)
assert list(shrunk_cov.index) == list(df.columns)
assert list(shrunk_cov.columns) == list(df.columns)
assert not shrunk_cov.isnull().any().any()
assert risk_models._is_positive_semidefinite(shrunk_cov)
def test_shrunk_covariance_extreme_delta():
df = get_data()
cs = risk_models.CovarianceShrinkage(df)
# if delta = 0, no shrinkage occurs
shrunk_cov = cs.shrunk_covariance(0)
np.testing.assert_array_almost_equal(shrunk_cov.values, risk_models.sample_cov(df))
# if delta = 1, sample cov does not contribute to shrunk cov
shrunk_cov = cs.shrunk_covariance(1)
N = df.shape[1]
F = np.identity(N) * np.trace(cs.S) / N
np.testing.assert_array_almost_equal(shrunk_cov.values, F * 252)
def test_efficient_return_shrunk():
df = get_data()
ef = setup_efficient_frontier()
ef.cov_matrix = risk_models.CovarianceShrinkage(df).ledoit_wolf(
shrinkage_target="single_factor")
w = ef.efficient_return(0.22)
assert isinstance(w, dict)
assert set(w.keys()) == set(ef.tickers)
np.testing.assert_almost_equal(ef.weights.sum(), 1)
assert all([i >= 0 for i in w.values()])
np.testing.assert_allclose(ef.portfolio_performance(),
(0.22, 0.0849639369932322, 2.353939884117318))
def build_portfolio(self, price_pivot, portfolio_total=10000):
''' build a portfolio from price data'''
mu = expected_returns.mean_historical_return(price_pivot)
shrink = risk_models.CovarianceShrinkage(price_pivot)
S = shrink.ledoit_wolf()
ef = EfficientFrontier(mu, S, weight_bounds=(0, 0.2), gamma=0.8)
weights = ef.max_sharpe()
weights = ef.clean_weights()
latest_prices = get_latest_prices(price_pivot)
weights = {k: v for k, v in weights.items() if weights[k] > 0.0}
da = DiscreteAllocation(weights, latest_prices, total_portfolio_value=portfolio_total)
allocation, leftover = da.lp_portfolio()
# print("Discrete allocation:", allocation)
return allocation
def get_weights(data,
start_date,
end_date,
columns,
n_threads=1,
fixed='sharpe',
value=0.05):
"""
:param data: DataFrame,
Full data without missing values, columns are identifiers
for different funds, index is datetime.
:param start_date: str
Start date to estimate mu and sigma.
:param end_date: str
End date to estimate mu and sigma.
:param columns: list, ['id1', 'id2',..., 'idn']
Identifiers of funds to use to optimize.
:param n_threads: int
Number of threads.
:param fixed: str, 'sharpe' or 'volatility' or 'return'
Optimization constraints, defaults to 'sharpe'.
:param value: float
Optimization constraints. No need to set if fixed is
'sharpe', defaults to 0.05.
:return: tuple, (float, float, float, dict)
The closest answer given fixed constraints.
(return, volatility, Sharpe ratio, weights)
weights == {'id1':w1, 'id2':w2, ..., 'idn':wn}
"""
while start_date not in data.index:
splits = start_date.split('-')
y, m, d = int(splits[0]), int(splits[1]), int(splits[2])
start_date = (datetime.datetime(y, m, d) +
datetime.timedelta(days=1)).strftime('%Y-%m-%d')
while end_date not in data.index:
splits = end_date.split('-')
y, m, d = int(splits[0]), int(splits[1]), int(splits[2])
end_date = (datetime.datetime(y, m, d) +
datetime.timedelta(days=-1)).strftime('%Y-%m-%d')
# print(start_date)
# print(end_date)
subdata = data.loc[start_date:end_date, columns]
optimizer = PortfolioOptimizer(
expected_returns.ema_historical_return(subdata),
risk_models.CovarianceShrinkage(subdata).ledoit_wolf())
optimizer.optimize(n_threads=n_threads)
return optimizer.get_fixed_ans(fixed, value)
def test_min_volatilty_shrunk_L2_reg():
df = get_data()
ef = setup_efficient_frontier()
ef.add_objective(objective_functions.L2_reg)
ef.cov_matrix = risk_models.CovarianceShrinkage(df).ledoit_wolf(
shrinkage_target="constant_correlation")
w = ef.min_volatility()
assert isinstance(w, dict)
assert set(w.keys()) == set(ef.tickers)
np.testing.assert_almost_equal(ef.weights.sum(), 1)
assert all([i >= 0 for i in w.values()])
np.testing.assert_allclose(
ef.portfolio_performance(),
(0.17358178582309983, 0.19563960638632416, 0.7850239972361532),
)
def weight(
self,
market_state: pd.DataFrame,
agent_portfolio: Dict[str, float],
is_recommendation: bool = False,
) -> Dict[str, float]:
# make default weight map
weights = {}
stocks = set(market_state.symbol.unique())
for stock in stocks:
weights[stock] = 0.0
current_date = market_state.date.max()
if current_date.year == market_state.date.min().year:
# there is no previous year
return weights
if not is_recommendation:
if current_date != market_state.loc[market_state.date.dt.year ==
current_date.year].date.min():
# it's not the first trading day of the current year
if current_date == self.all_dates[
(self.all_dates.date.dt.year == current_date.year)
& (self.all_dates.date.dt.month == 12)].date.max():
for stock in agent_portfolio.keys():
weights[stock] = -1
return weights
return weights
prices = market_state.pivot(values="adjusted_close",
index="date",
columns="symbol")
# Get Mu and sigma for efficient frontier
mu = expected_returns.mean_historical_return(prices)
sigma = risk_models.CovarianceShrinkage(prices).ledoit_wolf()
# Calculate efficient portfolio, objective: maximize sharpe ratio
ef = EfficientFrontier(mu, sigma, weight_bounds=(0, 1))
ef.max_sharpe()
cleaned_weights = ef.clean_weights()
print(cleaned_weights)
return dict(cleaned_weights)
def test_min_volatilty_shrunk_L2_reg():
df = get_data()
ef = setup_efficient_frontier()
ef.add_objective(objective_functions.L2_reg)
ef.cov_matrix = risk_models.CovarianceShrinkage(df).ledoit_wolf(
shrinkage_target="constant_correlation")
w = ef.min_volatility()
assert isinstance(w, dict)
assert set(w.keys()) == set(ef.tickers)
np.testing.assert_almost_equal(ef.weights.sum(), 1)
assert all([i >= 0 for i in w.values()])
np.testing.assert_allclose(
ef.portfolio_performance(),
(0.23127405291296832, 0.19563921371709164, 1.079916694096337),
)
def fit(self, prices: pd.DataFrame):
# TODO: make prices instance of portfolio
'''
Given prices finds wieghts (coefficients), most optimal portfolio given target function.
:param prices:
:return:
'''
self.frequency = autodetect_frequency(prices)
self.mu = expected_returns.mean_historical_return(
prices, frequency=self.frequency)
self.S = risk_models.CovarianceShrinkage(
prices, frequency=self.frequency).ledoit_wolf()
# self.S=risk_models.exp_cov(prices)
self.ef = EfficientFrontier(self.mu, self.S)
self.ef.add_objective(objective_functions.L2_reg)
getattr(self.ef, self.target_function)(**self.target_function_params)
self.coef_ = self.ef.clean_weights()
return self
def test_bl_relative_views():
df = get_data()
S = risk_models.CovarianceShrinkage(df).ledoit_wolf()
# 1. SBUX will drop by 20%
# 2. GOOG outperforms FB by 10%
# 3. BAC and JPM will outperform T and GE by 15%
views = np.array([-0.20, 0.10, 0.15]).reshape(-1, 1)
picking = np.array([
[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1],
[1, 0, -1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, -0.5, 0, 0, 0.5, 0, -0.5, 0, 0, 0, 0, 0, 0, 0, 0.5, 0],
])
bl = BlackLittermanModel(S, Q=views, P=picking)
rets = bl.bl_returns()
assert rets["SBUX"] < 0
assert rets["GOOG"] > rets["FB"]
assert (rets["BAC"] > rets["T"]) and (rets["JPM"] > rets["GE"])
def test_bl_returns_all_views():
df = get_data()
prior = expected_returns.ema_historical_return(df)
S = risk_models.CovarianceShrinkage(df).ledoit_wolf()
views = pd.Series(0.1, index=S.columns)
bl = BlackLittermanModel(S, pi=prior, Q=views)
posterior_rets = bl.bl_returns()
assert isinstance(posterior_rets, pd.Series)
assert list(posterior_rets.index) == list(df.columns)
assert posterior_rets.notnull().all()
assert posterior_rets.dtype == "float64"
np.testing.assert_array_almost_equal(
posterior_rets,
np.array(
[
0.11774473,
0.1709139,
0.12180833,
0.21202423,
0.28120945,
-0.2787358,
0.17274774,
0.12714698,
0.25492005,
0.11229777,
0.07182723,
-0.01521839,
-0.21235465,
0.06399515,
-0.11738365,
0.28865661,
0.23828607,
0.12038049,
0.2331218,
0.10485376,
]
),
)
def test_bl_returns_all_views():
df = get_data()
prior = expected_returns.ema_historical_return(df)
S = risk_models.CovarianceShrinkage(df).ledoit_wolf()
views = pd.Series(0.1, index=S.columns)
bl = BlackLittermanModel(S, pi=prior, Q=views)
posterior_rets = bl.bl_returns()
assert isinstance(posterior_rets, pd.Series)
assert list(posterior_rets.index) == list(df.columns)
assert posterior_rets.notnull().all()
assert posterior_rets.dtype == "float64"
np.testing.assert_array_almost_equal(
posterior_rets,
np.array(
[
0.11168648,
0.16782938,
0.12516799,
0.24067997,
0.32848296,
-0.22789895,
0.16311297,
0.11928542,
0.25414308,
0.11007738,
0.06282615,
-0.03140218,
-0.16977172,
0.05254821,
-0.10463884,
0.32173375,
0.26399864,
0.1118594,
0.22999558,
0.08977448,
]
),
)
def mean_var(my_portfolio, vol_max=0.15, perf=True) -> list:
# changed to take in desired timeline, the problem is that it would use all historical data
ohlc = yf.download(
my_portfolio.portfolio,
start=my_portfolio.start_date,
end=my_portfolio.end_date,
progress=False,
)
prices = ohlc["Adj Close"].dropna(how="all")
prices = prices.filter(my_portfolio.portfolio)
# sometimes we will pick a date range where company isn't public we can't set price to 0 so it has to go to 1
prices = prices.fillna(1)
mu = expected_returns.capm_return(prices)
S = risk_models.CovarianceShrinkage(prices).ledoit_wolf()
ef = EfficientFrontier(mu, S)
ef.add_objective(objective_functions.L2_reg,
gamma=my_portfolio.diversification)
if my_portfolio.min_weights is not None:
ef.add_constraint(lambda x: x >= my_portfolio.min_weights)
if my_portfolio.max_weights is not None:
ef.add_constraint(lambda x: x <= my_portfolio.max_weights)
ef.efficient_risk(vol_max)
weights = ef.clean_weights()
wts = weights.items()
result = []
for val in wts:
a, b = map(list, zip(*[val]))
result.append(b)
if perf is True:
ef.portfolio_performance(verbose=True)
return flatten(result)
def calculateInvestment(limit=10,
count=10,
write_to_file=True,
show_cla=False,
tpv=20000):
symbols = getSymbolsFromDatabase()
prices = createDataFrame(symbols[:limit], count)
mu = expected_returns.mean_historical_return(prices)
S = risk_models.CovarianceShrinkage(prices).ledoit_wolf()
ef = EfficientFrontier(mu, S, weight_bounds=(-1, 1))
ef.add_objective(objective_functions.L2_reg)
ef.min_volatility()
c_weights = ef.clean_weights()
if write_to_file == True:
ef.save_weights_to_file("weights.txt")
if show_cla == True:
cla = CLA(mu, S)
ef_plot(cla)
ef.portfolio_performance(verbose=True)
latest_prices = disc_alloc.get_latest_prices(prices)
allocation_minv, leftover = disc_alloc.DiscreteAllocation(
c_weights, latest_prices, total_portfolio_value=tpv).lp_portfolio()
return allocation_minv, leftover
def test_ledoit_wolf_raises_not_implemented():
df = get_data()
cs = risk_models.CovarianceShrinkage(df)
with pytest.raises(NotImplementedError):
cs.ledoit_wolf(shrinkage_target="I have not been implemented!")
def test_covariance_shrinkage_init():
df = get_data()
cs = risk_models.CovarianceShrinkage(df)
assert cs.S.shape == (20, 20)
assert not (np.isnan(cs.S)).any()
downside = long_df.loc[long_df["foxInstrumentName"] == i]["downsidePrice"].values[0]
down[i] = downside
exp_ret_upside[i] = ((upside + carry) - current) / current
skew[i] = (((upside + carry) - current) / current) / (
(current - (downside)) / current
)
exp_ret = pd.Series(exp_ret_upside)
skw = pd.Series(skew)
#%%
sector_upper["Oil & Oil Services"] = 0.25
sector_upper["Consumer & Retail"] = 0.25
mu = expected_returns.mean_historical_return(df)
S = risk_models.CovarianceShrinkage(df).ledoit_wolf()
#plotting.plot_covariance(S)
corr = df.corr()
layout = go.Layout(title='Correlation Heatmap',
xaxis=dict(tickfont = dict(size = 6)
),
yaxis=dict(tickfont = dict(size = 6)
),
showlegend=True,
)
fig = go.Figure(data=go.Heatmap(
z=corr,
x=df.columns,
