def demo(n, y, cov_fun, loo, k, ylim, figsize, seed):
"""Simple demo showing the results of the various shrinkage methods"""
np.random.seed(seed)
T = y * n
Sigma, tau = _cov_functions[cov_fun](n)
X = sample(Sigma, T)
S = cov(X)
L = eig(S, return_eigenvectors=False)
lam_1, lam_N = L[0], L[-1]
fig, (ax0, ax1) = plt.subplots(figsize=figsize, ncols=2)
ax0.plot(annualize_vol(tau / n), label='true')
ax1.plot(annualize_vol(tau / n), label='true')
ax0.plot(annualize_vol(L / n), label='sample')
ax1.plot(annualize_vol(L / n), label='sample')
# Oracle LW NLS shrinkage
_, d_lw_oracle = nls_oracle(X, S, Sigma)
d_isolw_oracle = isotonic_regression(d_lw_oracle)
ax0.plot(annualize_vol(d_lw_oracle / n), label='lw oracle')
ax1.plot(annualize_vol(d_isolw_oracle / n), label='lw oracle')
# LW NLS shrinkage
S_lw = nlshrink_covariance(X, centered=True)
d_lw = eig(S_lw, return_eigenvectors=False)
ax1.plot(annualize_vol(d_lw / n), label='lw')
if loo:
# LOO LW NLS shrinkage
_, d_loo = nls_loo_cv(X, S)
d_isoloo = isotonic_regression(d_loo)
ax0.plot(annualize_vol(d_loo / n), label='noisy-loo')
ax1.plot(annualize_vol(d_isoloo / n), label='isoloo')
# K-fold LW NLS shrinkage
_, d_kfold = nls_kfold_cv(X, S, k)
d_isokfold = isotonic_regression(d_kfold)
ax0.plot(annualize_vol(d_kfold / n), label='noisy-kfold')
ax1.plot(annualize_vol(d_isokfold / n), label='isokfold')
# MinVar NLS shrinkage
_, d_mv_oracle = minvar_nls_oracle(X, S, Sigma)
d_isomv_oracle = isotonic_regression(d_mv_oracle, y_min=lam_N, y_max=lam_1)
_, d_isolsq_mv_oracle = minvar_nls_oracle(X, S, Sigma, isotonic=True)
ax0.plot(annualize_vol(d_mv_oracle / n), label='noisy-mv_oracle')
ax1.plot(annualize_vol(d_isomv_oracle / n), label='buggy-iso-mv_oracle')
ax1.plot(annualize_vol(d_isolsq_mv_oracle / n), label='isolsq-mv_oracle')
ax0.legend()
ax1.legend()
ax0.set_ylim(*ylim)
ax1.set_ylim(*ylim)
plt.show()
def var_ratio(M, N, y, cov_fun, gamma, lw, loo, K, ylim, figsize, seed):
"""
Generate box plots of the variance ratios from the various shrinkage
methods for the minimum variance portfolio (global or long-only depending
on option flag).
Leave-One-Out cross-validated NLS and the optimal LW NLS are not turned on
by default since they take a while to compute and 10-Fold CV NLS w/ isotonic
regression is fast and just as good in terms of accuracy.
"""
np.random.seed(seed)
T = y * N
names = [
'sample', 'lw_oracle', 'lw_iso_oracle', 'lw_kfold', 'lw_isokfold',
'mv_isonlsq_oracle', 'mv_isonlsq_kfold'
] # , 'isomv_oracle']
if loo:
names.insert(1, 'lw_loo')
names.insert(2, 'lw_isoloo')
if lw:
names.insert(1, 'lw')
empty_df = pd.DataFrame(np.zeros((M, len(names))), columns=names)
dfs = {
'oos_var': empty_df,
'is_var': empty_df.copy(),
'forecast_var_ratio': empty_df.copy(),
'true_var_ratio': empty_df.copy(),
'te': empty_df.copy(),
}
# forecast_var_ratio_df = pd.DataFrame(
# np.zeros((M, len(names))), columns=names)
# oos_var_df = pd.DataFrame(np.zeros((M, len(names))), columns=names)
# is_var_df = pd.DataFrame(np.zeros((M, len(names))), columns=names)
# true_var_ratio_df = pd.DataFrame(np.zeros((M, len(names))), columns=names)
# te_df = pd.DataFrame(np.zeros((M, len(names))), columns=names)
pbar = tqdm(total=M)
results = []
for j in range(M):
# Build Model
if cov_fun in ['slr', 'factor']:
fm_seed = np.random.randint(1, 2**32 - 1)
Sigma, tau = cov_functions[cov_fun](N, seed=fm_seed)
else:
Sigma, tau = cov_functions[cov_fun](N)
pi_true = min_var_portfolio(Sigma, gamma=gamma)
# Generate Data
X = sample(Sigma, T)
X = X - X.mean()
S = cov(X)
lam, U = eig(S)
# Sample covariance
name = 'sample'
result = portfolio_analysis(S, Sigma, gamma, pi_true)
results.append({name: result})
for key in result:
dfs[key].loc[j, name] = result[key]
# Oracle LW NLS shrinkage
name = 'lw_oracle'
_, d_lw_oracle = nls_oracle(X, S, U, Sigma)
S_lw_oracle = eig_multiply(U, d_lw_oracle)
result = portfolio_analysis(S_lw_oracle, Sigma, gamma, pi_true)
results.append({name: result})
for key in result:
dfs[key].loc[j, name] = result[key]
name = 'lw_iso_oracle'
d_lw_iso_oracle = isotonic_regression(d_lw_oracle)
S_lw_iso_oracle = eig_multiply(U, d_lw_iso_oracle)
result = portfolio_analysis(S_lw_iso_oracle, Sigma, gamma, pi_true)
results.append({name: result})
for key in result:
dfs[key].loc[j, name] = result[key]
# LW NLS shrinkage
if lw:
name = 'lw'
S_lw = nlshrink_covariance(X, centered=True)
result = portfolio_analysis(S_lw, Sigma, gamma, pi_true)
results.append({name: result})
for key in result:
dfs[key].loc[j, name] = result[key]
# LOO LW NLS shrinkage
if loo:
name = 'lw_loo'
_, d_lw_loo = nls_loo_cv(X, S, U)
S_lw_loo = eig_multiply(U, d_lw_loo)
result = portfolio_analysis(S_lw_loo, Sigma, gamma, pi_true)
results.append({name: result})
for key in result:
dfs[key].loc[j, name] = result[key]
name = 'lw_isoloo'
d_lw_isoloo = isotonic_regression(d_lw_loo)
S_lw_isoloo = eig_multiply(U, d_lw_isoloo)
result = portfolio_analysis(S_lw_isoloo, Sigma, gamma, pi_true)
results.append({name: result})
for key in result:
dfs[key].loc[j, name] = result[key]
# K-fold LW NLS shrinkage
name = 'lw_kfold'
_, d_lw_kfold = nls_kfold_cv(X, S, U, K)
S_lw_kfold = eig_multiply(U, d_lw_kfold)
result = portfolio_analysis(S_lw_kfold, Sigma, gamma, pi_true)
results.append({name: result})
for key in result:
dfs[key].loc[j, name] = result[key]
name = 'lw_isokfold'
d_lw_isokfold = isotonic_regression(d_lw_kfold)
S_lw_isokfold = eig_multiply(U, d_lw_isokfold)
result = portfolio_analysis(S_lw_isokfold, Sigma, gamma, pi_true)
results.append({name: result})
for key in result:
dfs[key].loc[j, name] = result[key]
# MinVar NLS shrinkage
_, d_mv_oracle = minvar_nls_oracle(X, S, lam, U, Sigma)
# Note: the raw oracle values for MinVar shrinkage are likely to
# produce negative eigenvalues, which means the minimum variance
# portfolio cannot be reasonably computed. Computing variance
# ratios for the MinVar shrinkage only works with some kind of
# modification to the raw values.
# Note: Applying isotonic regression after solving for the oracle values
# is consistently way worse than solving the constrained LS problem so
# it is omitted.
# d_mv_iso_oracle = isotonic_regression(d_mv_oracle)
# S_mv_iso_oracle = eig_multiply(U, d_mv_iso_oracle)
name = 'mv_isonlsq_oracle'
_, d_mv_isonlsq_oracle = minvar_nls_oracle(X,
S,
lam,
U,
Sigma,
isotonic=True)
S_mv_isonlsq_oracle = eig_multiply(U, d_mv_isonlsq_oracle)
result = portfolio_analysis(S_mv_isonlsq_oracle, Sigma, gamma, pi_true)
results.append({name: result})
for key in result:
dfs[key].loc[j, name] = result[key]
name = 'mv_isonlsq_kfold'
_, d_mv_isonlsq_kfold = minvar_nls_kfold(X, S, lam, U, K)
S_d_mv_isonlsq_kfold = eig_multiply(U, d_mv_isonlsq_kfold)
result = portfolio_analysis(S_d_mv_isonlsq_kfold, Sigma, gamma,
pi_true)
results.append({name: result})
for key in result:
dfs[key].loc[j, name] = result[key]
pbar.update()
fig, ax = plt.subplots(figsize=figsize, ncols=5)
fig.suptitle("Shrinkage Performance: N={}".format(N))
dfs['forecast_var_ratio'].boxplot(ax=ax[0])
dfs['true_var_ratio'].boxplot(ax=ax[1])
dfs['oos_var'].boxplot(ax=ax[2])
dfs['is_var'].boxplot(ax=ax[3])
dfs['te'].boxplot(ax=ax[4])
ax[0].set_title('Forecast Variance Ratios')
ax[1].set_title('True Variance Ratios')
ax[2].set_title('Out-of-Sample Variance')
ax[3].set_title('In-Sample Variance')
ax[4].set_title('Tracking Error to True MinVar')
ax[0].set_ylim((0, 2.))
ax[1].set_ylim((0, 3.))
ylim = (.5 * min(dfs['is_var'].values.min(), dfs['oos_var'].values.min()),
2 * max(dfs['is_var'].values.max(), dfs['oos_var'].values.max()))
ax[2].set_ylim(ylim)
ax[3].set_ylim(ylim)
fig.autofmt_xdate(rotation=90)
fig.subplots_adjust(left=0.05,
right=0.95,
bottom=.22,
top=0.9,
wspace=.36,
hspace=.2)
plt.show()
def eigs(m, n, y, cov_fun, lw, loo, k, ylim, figsize, seed):
"""
Generate box plots of the eigenvalues from the various shrinkage methods.
Leave-One-Out cross-validated NLS and the optimal LW NLS are not turned on
by default since they take a while to compute and 10-Fold CV NLS w/ isotonic
regression is fast and just as good in terms of accuracy.
"""
# Setup covariance
np.random.seed(seed)
T = y * n
names = [
'true', 'sample', 'lw_oracle', 'isolw_oracle', 'kfold', 'isokfold',
'mv_oracle', 'isonlsq_mv_oracle', 'isonlsq_mv_kfold'
]
if lw:
names += ['lw']
if loo:
names += ['loo', 'isoloo']
dfs = {name: pd.DataFrame(np.zeros((m, n))) for name in names}
pbar = tqdm(total=m)
for j in range(m):
# Build Model
if cov_fun in ['slr', 'factor']:
fm_seed = np.random.randint(1, 2**32 - 1)
Sigma, tmp = cov_functions[cov_fun](n, seed=fm_seed)
else:
Sigma, tmp = cov_functions[cov_fun](n)
dfs['true'].iloc[j, :] = tau = annualize_vol(tmp / n)
if ylim is None:
ylim = (0., 2 * np.max(tau))
# Generate data
X = sample(Sigma, T)
S = cov(X)
lam, U = eig(S)
# Note: eigenvalues need to be scaled by 1 / n to convert to variance
# Sample covariance
dfs['sample'].iloc[j, :] = annualize_vol(lam / n)
# Oracle LW NLS shrinkage
_, tmp = nls_oracle(X, S, U, Sigma)
dfs['lw_oracle'].iloc[j, :] = annualize_vol(tmp / n)
tmp = isotonic_regression(tmp)
dfs['isolw_oracle'].iloc[j, :] = annualize_vol(tmp / n)
# LW NLS shrinkage
if lw:
S_lw = nlshrink_covariance(X, centered=True)
tmp = eig(S_lw, return_eigenvectors=False)
dfs['lw'].loc[j, :] = annualize_vol(tmp / n)
# LOO LW NLS shrinkage
if loo:
_, tmp = nls_loo_cv(X, S, U)
dfs['loo'].iloc[j, :] = annualize_vol(tmp / n)
tmp = isotonic_regression(tmp)
dfs['isoloo'].iloc[j, :] = annualize_vol(tmp / n)
# K-fold LW NLS shrinkage
_, tmp = nls_kfold_cv(X, S, U, k)
dfs['kfold'].iloc[j, :] = annualize_vol(tmp / n)
tmp = isotonic_regression(tmp)
dfs['isokfold'].iloc[j, :] = annualize_vol(tmp / n)
# MinVar NLS shrinkage
_, tmp = minvar_nls_oracle(X, S, lam, U, Sigma)
dfs['mv_oracle'].iloc[j, :] = annualize_vol(tmp / n)
# Note: Applying isotonic regression after solving for the oracle values
# is consistently way worse than solving the constrained LS problem so
# it is omitted.
# lam_1, lam_n = lam[0], lam[-1]
# tmp = isotonic_regression(tmp, y_min=lam_n, y_max=lam_1)
# dfs['isomv_oracle'].iloc[j, :] = annualize_vol(tmp / n)
_, tmp = minvar_nls_oracle(X, S, lam, U, Sigma, isotonic=True)
dfs['isonlsq_mv_oracle'].iloc[j, :] = annualize_vol(tmp / n)
_, tmp = minvar_nls_kfold(X, S, lam, U, k)
dfs['isonlsq_mv_kfold'].iloc[j, :] = annualize_vol(tmp / n)
pbar.update()
# Generate band plots for various shrinkage methods
fig, (ax0, ax1, ax2) = plt.subplots(figsize=figsize, ncols=3)
band_plot(dfs['true'], ax0, 'true')
band_plot(dfs['true'], ax1, 'true')
band_plot(dfs['true'], ax2, 'true')
band_plot(dfs['sample'], ax0, 'sample')
band_plot(dfs['sample'], ax1, 'sample')
band_plot(dfs['sample'], ax2, 'sample')
if lw:
band_plot(dfs['lw'], ax1, 'lw')
if loo:
band_plot(dfs['loo'], ax0, 'loo')
band_plot(dfs['isoloo'], ax1, 'isoloo')
band_plot(dfs['kfold'], ax0, 'kfold')
band_plot(dfs['isokfold'], ax1, 'isokfold')
band_plot(dfs['mv_oracle'], ax0, 'mv_oracle')
# band_plot(dfs['isomv_oracle'], ax1, 'isomv_oracle')
band_plot(dfs['isonlsq_mv_oracle'], ax2, 'isonlsq_mv_oracle')
band_plot(dfs['isonlsq_mv_kfold'], ax2, 'isonlsq_mv_kfold')
ax0.legend()
ax1.legend()
ax2.legend()
ax0.set_ylim(*ylim)
ax1.set_ylim(*ylim)
ax2.set_ylim(*ylim)
plt.show()
def var_ratio(m, n, y, cov_fun, gamma, lw, loo, k, ylim, figsize, seed):
"""
Generate box plots of the variance ratios from the various shrinkage
methods for the minimum variance portfolio (global or long-only depending
on option flag).
Leave-One-Out cross-validated NLS and the optimal LW NLS are not turned on
by default since they take a while to compute and 10-Fold CV NLS w/ isotonic
regression is fast and just as good in terms of accuracy.
"""
np.random.seed(seed)
T = y * n
Sigma, tau = _cov_functions[cov_fun](n)
names = [
'sample', 'lw_oracle', 'isolw_oracle', 'kfold', 'isokfold',
'isolsq_mv_oracle'
] # , 'isomv_oracle']
if loo:
names.insert(1, 'loo')
names.insert(2, 'isoloo')
if lw:
names.insert(1, 'lw')
var_ratio_df = pd.DataFrame(np.zeros((m, len(names))), columns=names)
oos_vol_df = pd.DataFrame(np.zeros((m, len(names))), columns=names)
pbar = tqdm(total=m)
for j in range(m):
X = sample(Sigma, T)
X = X - X.mean()
S = cov(X)
L, U = eig(S)
# Sample covariance
pi_S = min_var_portfolio(S, gamma=gamma)
oos_vol_df.loc[j, 'sample'] = portfolio_vol(pi_S, Sigma)
var_ratio_df.loc[j, 'sample'] = variance_ratio(pi_S, S, Sigma)
# Oracle LW NLS shrinkage
_, d_lw_oracle = nls_oracle(X, S, Sigma)
S_lw_oracle = eig_multiply(U, d_lw_oracle)
pi_lw_oracle = min_var_portfolio(S_lw_oracle, gamma=gamma)
oos_vol_df.loc[j, 'lw_oracle'] = portfolio_vol(pi_lw_oracle, Sigma)
var_ratio_df.loc[j,
'lw_oracle'] = variance_ratio(pi_lw_oracle,
S_lw_oracle, Sigma)
d_isolw_oracle = isotonic_regression(d_lw_oracle)
S_isolw_oracle = eig_multiply(U, d_isolw_oracle)
pi_isolw_oracle = min_var_portfolio(S_isolw_oracle, gamma=gamma)
oos_vol_df.loc[j,
'isolw_oracle'] = portfolio_vol(pi_isolw_oracle, Sigma)
var_ratio_df.loc[j, 'isolw_oracle'] = variance_ratio(
pi_isolw_oracle, S_isolw_oracle, Sigma)
# LW NLS shrinkage
if lw:
S_lw = nlshrink_covariance(X, centered=True)
pi_lw = min_var_portfolio(S_lw, gamma=gamma)
var_ratio_df.loc[j, 'lw'] = variance_ratio(pi_lw, S_lw, Sigma)
# LOO LW NLS shrinkage
if loo:
_, d_loo = nls_loo_cv(X, S)
S_loo = eig_multiply(U, d_loo)
pi_loo = min_var_portfolio(S_loo, gamma=gamma)
oos_vol_df.loc[j, 'loo'] = portfolio_vol(pi_loo, Sigma)
var_ratio_df.loc[j, 'loo'] = variance_ratio(pi_loo, S_loo, Sigma)
d_isoloo = isotonic_regression(d_loo)
S_isoloo = eig_multiply(U, d_isoloo)
pi_isoloo = min_var_portfolio(S_isoloo, gamma=gamma)
oos_vol_df.loc[j, 'isoloo'] = portfolio_vol(pi_isoloo, Sigma)
var_ratio_df.loc[j, 'isoloo'] = variance_ratio(
pi_isoloo, S_isoloo, Sigma)
# K-fold LW NLS shrinkage
_, d_kfold = nls_kfold_cv(X, S, k)
S_kfold = eig_multiply(U, d_kfold)
pi_kfold = min_var_portfolio(S_kfold, gamma=gamma)
oos_vol_df.loc[j, 'kfold'] = portfolio_vol(pi_kfold, Sigma)
var_ratio_df.loc[j, 'kfold'] = variance_ratio(pi_kfold, S_kfold, Sigma)
d_isokfold = isotonic_regression(d_kfold)
S_isokfold = eig_multiply(U, d_isokfold)
pi_isokfold = min_var_portfolio(S_isokfold, gamma=gamma)
oos_vol_df.loc[j, 'isokfold'] = portfolio_vol(pi_isokfold, Sigma)
var_ratio_df.loc[j,
'isokfold'] = variance_ratio(pi_isokfold, S_isokfold,
Sigma)
# MinVar NLS shrinkage
_, d_mv_oracle = minvar_nls_oracle(X, S, Sigma)
# Note: the raw oracle values for MinVar shrinkage are likely to
# produce negative eigenvalues, which means the minimum variance
# portfolio cannot be reasonably computed. Computing variance
# ratios for the MinVar shrinkage only works with some kind of
# modification to the raw values.
# Note: Applying isotonic regression after solving for the oracle values
# is consistently way worse than solving the constrained LS problem so
# it is omitted.
# lam_1, lam_n = L[0], L[-1]
# d_isomv_oracle = isotonic_regression(
# d_mv_oracle, y_min=lam_n, y_max=lam_1)
# S_isomv_oracle = eig_multiply(U, d_isomv_oracle)
# pi_isomv_oracle = min_var_portfolio(S_isomv_oracle, gamma=gamma)
# oos_vol_df.loc[j, 'isomv_oracle'] = portfolio_vol(
# pi_isomv_oracle, Sigma)
# var_ratio_df.loc[j, 'isomv_oracle'] = variance_ratio(
# pi_isomv_oracle, S_isomv_oracle, Sigma)
_, d_isolsq_mv_oracle = minvar_nls_oracle(X, S, Sigma, isotonic=True)
S_isolsq_mv_oracle = eig_multiply(U, d_isolsq_mv_oracle)
pi_isolsq_mv_oracle = min_var_portfolio(S_isolsq_mv_oracle,
gamma=gamma)
oos_vol_df.loc[j, 'isolsq_mv_oracle'] = portfolio_vol(
pi_isolsq_mv_oracle, Sigma)
var_ratio_df.loc[j, 'isolsq_mv_oracle'] = variance_ratio(
pi_isolsq_mv_oracle, S_isolsq_mv_oracle, Sigma)
pbar.update()
fig, ax = plt.subplots(figsize=figsize, ncols=2)
var_ratio_df.boxplot(ax=ax[0])
oos_vol_df.boxplot(ax=ax[1])
# ax.legend()
plt.show()
def nls_asymptotic(sim, isotonic=False):
X = sim.X
S_lw = nlshrink_covariance(X, centered=True)
d = eig(S_lw, return_eigenvectors=False)
return isotonic_regression(d) if isotonic else d
def nls_asymptotic(X, S, U):
S_lw = nlshrink_covariance(X, centered=True)
d_lw = eig(S_lw, return_eigenvectors=False)
return U, d_lw