# S&P 500 index value spx_path = '../../../databases/global-databases/equities/db_stocks_SP500/SPX.csv' spx_all = pd.read_csv(spx_path, parse_dates=['date']) spx = spx_all.loc[(spx_all['date'] >= pd.to_datetime('2004-01-02')) & (spx_all['date'] < pd.to_datetime('2017-09-01'))] # ## [Step 1](https://www.arpm.co/lab/redirect.php?permalink=s_exp_decay_fp-implementation-step01): Compute the S&P 500 compounded return # invariants (S&P500 log-return) epsi = np.diff(np.log(spx.SPX_close)) # S&P 500 index compounded return # ## [Step 2](https://www.arpm.co/lab/redirect.php?permalink=s_exp_decay_fp-implementation-step02): Compute the time exponential decay probabilities t_ = len(epsi) t_star = t_ p_exp = exp_decay_fp(t_, tau_hl, t_star) # ## [Step 3](https://www.arpm.co/lab/redirect.php?permalink=s_exp_decay_fp-implementation-step03): Compute the effective number of scenarios ens = effective_num_scenarios(p_exp) # ## [Step 4](https://www.arpm.co/lab/redirect.php?permalink=s_exp_decay_fp-implementation-step04): Compute flexible probabilities histogram f_eps, x_eps = histogram_sp(epsi, p=p_exp, k_=10*np.log(t_)) # ## Plots # + # figure settings plt.style.use('arpm') grey_range = np.r_[np.arange(0, 0.6 + 0.01, 0.01), .85]
df_stocks = df_stocks.loc[(df_stocks.index >= t_first) & (df_stocks.index <= t_last)] # remove the stocks with missing values df_stocks = df_stocks.dropna(axis=1, how='any') # - # ## [Step 1](https://www.arpm.co/lab/redirect.php?permalink=s_dcc_fit-implementation-step01): Compute log-returns v_stock = np.array(df_stocks.iloc[:, :n_]) dx = np.diff(np.log(v_stock), axis=0) # S&P 500 index compounded return t_ = dx.shape[0] # ## [Step 2](https://www.arpm.co/lab/redirect.php?permalink=s_dcc_fit-implementation-step02): Set flexible probabilities p = exp_decay_fp(t_, tau_hl) # flexible probabilities # ## [Step 3](https://www.arpm.co/lab/redirect.php?permalink=s_dcc_fit-implementation-step03): Fit a GARCH(1,1) on each time series of compounded returns param = np.zeros((4, n_)) sigma2 = np.zeros((t_, n_)) xi = np.zeros((t_, n_)) for n in range(n_): param[:, n], sigma2[:, n], xi[:, n] = \ fit_garch_fp(dx[:, n], p, rescale=True) # ## [Step 4](https://www.arpm.co/lab/redirect.php?permalink=s_dcc_fit-implementation-step04): Estimate marginal distributions by fitting a Student t distribution via MLFP mu_marg = np.zeros(n_) sigma2_marg = np.zeros(n_) for n in range(n_):
# values v = db_stocks_sp.values vix = db_vix.values[:, 0] # - # ## [Step 1](https://www.arpm.co/lab/redirect.php?permalink=s_fit_garch_stocks-implementation-step01): Risk drivers identification x = np.log(v) # log-values d_ = x.shape[1] # ## [Step 2](https://www.arpm.co/lab/redirect.php?permalink=s_fit_garch_stocks-implementation-step02): Quest for invariance # + i_ = d_ epsi = np.zeros((t_, i_)) p_garch = exp_decay_fp(t_, tau_hl_garch) for i in range(i_): print('Fitting ' + str(i+1) + '-th GARCH; ' + str(int((i+1)/i_*100)) + '% done.') _, _, epsi[:, i] = fit_garch_fp(np.diff(x[:, i], axis=0), p_garch) # - # ## [Step 3](https://www.arpm.co/lab/redirect.php?permalink=s_fit_garch_stocks-implementation-step03): Historical estimation # + # time and state conditioning on smoothed and scored VIX returns # state indicator: VIX compounded return realizations c_vix = np.diff(np.log(vix)) # smoothing
# + # time and state conditioning on smoothed and scored VIX returns # state indicator: VIX compounded return realizations db_vix['c_vix'] = np.log(db_vix).diff() # extract data for analysis dates c_vix = db_vix.c_vix[dates].values # smoothing z_smooth = smoothing(c_vix, tau_hl_smooth) # scoring z = scoring(z_smooth, tau_hl_score) # target value z_star = z[-1] # prior probabilities p_prior = exp_decay_fp(t_, tau_hl_prior) # posterior probabilities p = conditional_fp(z, z_star, alpha, p_prior) # effective number of scenarios ens = effective_num_scenarios(p) print('Effective number of scenarios is', int(round(ens))) # - # ## [Step 2](https://www.arpm.co/lab/redirect.php?permalink=s_checklist_scenariobased_step03-implementation-step02): Estimate the marginal distributions for stocks, S&P 500 and implied volatility # + # invariants to be modeled parametrically ind_parametric = np.arange(n_stocks + 1 + d_implvol, n_stocks + 1 + d_implvol + i_bonds) # invariants to be modeled nonparametrically
w = np.array([1 / 3, 1 / 3, 1 / 3]) # market-weighted portfolio # ## [Step 0](https://www.arpm.co/lab/redirect.php?permalink=s_bl_equilibrium_ret-implementation-step00): Load data path = '../../../databases/global-databases/equities/db_stocks_SP500/' data = pd.read_csv(path + 'db_stocks_sp.csv', index_col=0, header=[0, 1]) # ## [Step 1](https://www.arpm.co/lab/redirect.php?permalink=s_bl_equilibrium_ret-implementation-step01): Compute time series of returns n_ = len(w) # market dimension r_t = data.pct_change().iloc[1:, :n_].values # returns # ## [Step 2](https://www.arpm.co/lab/redirect.php?permalink=s_bl_equilibrium_ret-implementation-step02): Compute the sample mean and the exponential decay sample covariance t_ = len(r_t) p_t_tau_hl = exp_decay_fp(t_, tau_hl) # exponential decay probabilities mu_hat_r, sig2_hat_r = meancov_sp(r_t, p_t_tau_hl) # sample mean and cov. # ## [Step 3](https://www.arpm.co/lab/redirect.php?permalink=s_bl_equilibrium_ret-implementation-step03): Compute Black-Litterman prior parameters # + # expectation in terms of market equilibrium mu_r_equil = 2 * lam * sig2_hat_r @ w tau = t_ # uncertainty level in the reference model mu_m_pri = mu_r_equil cv_pri_pred = (1 + 1 / tau) * sig2_hat_r # - # ## [Step 4](https://www.arpm.co/lab/redirect.php?permalink=s_bl_equilibrium_ret-implementation-step04): Compute vectors quantifying the views
# ## [Step 1](https://www.arpm.co/lab/redirect.php?permalink=s_stock_selection-implementation-step01): Compute linear returns of both benchmark and securities # + # stocks return r_stock = np.diff(v_stock, axis=0) / v_stock[:-1, :] # S&P500 index return r_sandp = np.diff(v_sandp, axis=0) / v_sandp[:-1] # - # ## [Step 2](https://www.arpm.co/lab/redirect.php?permalink=s_stock_selection-implementation-step02): Cov. matrix of the joint vector of stocks and bench. returns # + # exponential decay probabilities p = exp_decay_fp(t_ - 1, tau_hl) # HFP covariance _, s2_r_stock_r_sandp = meancov_sp( np.concatenate((r_stock, r_sandp.reshape(-1, 1)), axis=1), p) # - # ## [Step 3](https://www.arpm.co/lab/redirect.php?permalink=s_stock_selection-implementation-step03): Objective function g = lambda s: obj_tracking_err(s2_r_stock_r_sandp, s)[1] # ## [Step 4](https://www.arpm.co/lab/redirect.php?permalink=s_stock_selection-implementation-step04): Portfolio selection via naive routine s_star_naive = naive_selection(g, n_, k_) g_te_naive = np.zeros(k_) for k in np.arange(0, k_):
# - # ## [Input parameters](https://www.arpm.co/lab/redirect.php?permalink=s_shrinkage_factor-parameters) tau_hl = 180 # half life k_ = 25 # number of hidden factors # ## [Step 0](https://www.arpm.co/lab/redirect.php?permalink=s_shrinkage_factor-implementation-step00): Load data path = '../../../databases/temporary-databases/' xi = np.array(pd.read_csv(path + 'db_GARCH_residuals.csv', index_col=0)) t_, n_ = xi.shape # ## [Step 1](https://www.arpm.co/lab/redirect.php?permalink=s_shrinkage_factor-implementation-step01): Compute the HFP correlation p = exp_decay_fp(t_, tau_hl) _, sigma2 = meancov_sp(xi, p) c2, _ = cov_2_corr(sigma2) # ## [Step 2](https://www.arpm.co/lab/redirect.php?permalink=s_shrinkage_factor-implementation-step02): Compute the loadings and residual variances via PAF factor analysis beta, delta2 = factor_analysis_paf(c2, k_) # ## [Step 3](https://www.arpm.co/lab/redirect.php?permalink=s_shrinkage_factor-implementation-step03): Compute the factor analysis correlation and the Frobenius norm c2_paf = beta @ beta.T + np.diag(delta2) d_fro = np.linalg.norm(c2 - c2_paf, ord='fro') / \ np.linalg.norm(c2, ord='fro') * 100. # ## Plots
# - # ## [Step 2](https://www.arpm.co/lab/redirect.php?permalink=s_market_prediction_regression-implementation-step02): Historical estimation # + # time and state conditioning on smoothed and scored VIX returns # state indicator: VIX compounded return realizations c_vix = np.diff(np.log(vix)) # smoothing z_vix = smoothing(c_vix, tau_hl_smooth) # scoring z_vix = scoring(z_vix, tau_hl_score) # target value z_vix_star = z_vix[-1] # flexible probabilities p_base = exp_decay_fp(len(dates), tau_hl_pri) p = conditional_fp(z_vix, z_vix_star, alpha_leeway, p_base) # HFP location and dispersion mu_hat, sig2_hat = meancov_sp(x, p) _, sig2_z_hat = meancov_sp(z, p) # OLS loadings _, beta_ols, sig2_u_ols, _ = fit_lfm_ols(x, z, p, fit_intercept=False) r2_ols = multi_r2(sig2_u_ols, sig2_hat) # - # ## [Step 3](https://www.arpm.co/lab/redirect.php?permalink=s_market_prediction_regression-implementation-step03): Maximum likelihood - GLM with normal assumption # + alpha_mlfp_norm, beta_mlfp_norm, sig2_u_mlfp_norm, _ = \
# select data within the date range df_stocks = df_stocks.loc[df_stocks.index].tail(t_) # select stock df_stocks = df_stocks['AMZN'] # stock value # - # ## [Step 1](https://www.arpm.co/lab/redirect.php?permalink=s_projection_brownian_motion-implementation-step01): Compute the risk driver x = np.log(np.array(df_stocks)) # log-value # ## [Step 2](https://www.arpm.co/lab/redirect.php?permalink=s_projection_brownian_motion-implementation-step02): Compute HFP mean and covariance epsi = np.diff(x) # invariant past realizations p = exp_decay_fp(t_ - 1, tau_hl) # exponential decay probabilities mu_hat, sig2_hat = meancov_sp(epsi, p) # HFP mean and covariance # ## [Step 3](https://www.arpm.co/lab/redirect.php?permalink=s_projection_brownian_motion-implementation-step03): Compute Monte Carlo paths of risk drivers x_tnow_thor = simulate_bm(x[-1].reshape(1), delta_t_m, mu_hat.reshape(1), sig2_hat.reshape((1, 1)), j_).squeeze() # ## [Step 4](https://www.arpm.co/lab/redirect.php?permalink=s_projection_brownian_motion-implementation-step04): Compute projected expectations and standard deviations mu_thor = x[-1] + mu_hat * np.cumsum(delta_t_m) # projected expectations sig_thor = np.sqrt(sig2_hat) * np.sqrt( np.cumsum(delta_t_m)) # projected standard deviations # ## Plots
# - # ## [Step 1](https://www.arpm.co/lab/redirect.php?permalink=s_lasso_vs_ridge-implementation-step01): Select stocks and SPX from database v_stocks = np.array(spx_stocks.iloc[:, 1 + np.arange(k_)]) # select stocks v_spx = np.array(spx_stocks.iloc[:, 0]) # ## [Step 2](https://www.arpm.co/lab/redirect.php?permalink=s_lasso_vs_ridge-implementation-step02): Compute linear returns of both SPX and stocks x = np.diff(v_spx) / v_spx[:-1] # benchmark z = np.diff(v_stocks, axis=0) / v_stocks[:-1, :] # factors t_ = len(x) # ## [Step 3](https://www.arpm.co/lab/redirect.php?permalink=s_lasso_vs_ridge-implementation-step03): Set the flexible probabilities p = exp_decay_fp(t_, tau_hl) # exponential decay # ## [Step 4](https://www.arpm.co/lab/redirect.php?permalink=s_lasso_vs_ridge-implementation-step04): Perform ridge regression lambdagrid_ridge = np.linspace(0, lambda_ridge_max, l_) # grid of penalties beta_r = np.zeros((k_, l_)) for l in range(l_): # ridge regression _, beta_r[:, l], _, _ = fit_lfm_ridge(x, z, p, lambdagrid_ridge[l]) # ## [Step 5](https://www.arpm.co/lab/redirect.php?permalink=s_lasso_vs_ridge-implementation-step05): Perform lasso regression lambdagrid_lasso = np.linspace(0, lambda_lasso_max, l_) # grid of penalties beta_l = np.zeros((k_, l_)) for l in range(l_): # lasso regression
# select data df_stocks = df_stocks[stock].tail(t_) # - # ## [Step 1](https://www.arpm.co/lab/redirect.php?permalink=s_projection_stock_bootstrap-implementation-step01): Compute risk driver # + x = np.log(np.array(df_stocks)) # log-value # - # ## [Step 2](https://www.arpm.co/lab/redirect.php?permalink=s_projection_stock_bootstrap-implementation-step02): HFP distribution of the invariant # + epsi = np.diff(x) # historical scenarios p = exp_decay_fp(t_ - 1, tau_hl) # probabilities # - # ## [Step 3](https://www.arpm.co/lab/redirect.php?permalink=s_projection_stock_bootstrap-implementation-step03): Generate scenarios of log-value via bootstrapping # + x_tnow_thor = simulate_rw_hfp(x[-1].reshape(1), epsi, p, j_, m_).squeeze() # - # ## [Step 4](https://www.arpm.co/lab/redirect.php?permalink=s_projection_stock_bootstrap-implementation-step04): Evolution of expectation and standard deviation # + mu_thor = np.zeros(m_ + 1) sig_thor = np.zeros(m_ + 1) for m in range(0, m_ + 1): mu_thor[m], sig2_thor = meancov_sp(x_tnow_thor[:, m].reshape(-1, 1))
# + y = np.array(df_y[tau.astype('str')]) # yields to maturity if y.shape[0] > t_: y = y[-t_:, :] else: t_ = y.shape[0] # increments dy = np.diff(y, 1, axis=0) # t_ennd-1 increments n_ = dy.shape[1] # - # ## [Step 2](https://www.arpm.co/lab/redirect.php?permalink=s_kalman_filter_yield_curve-implementation-step02): Set flexible probabilities and compute effective number of scenarios p = exp_decay_fp(dy.shape[0], tau_p) p = p / np.sum(p) # flexible probabilities ens = effective_num_scenarios(p) # effective number of scenarios # ## [Step 3](https://www.arpm.co/lab/redirect.php?permalink=s_kalman_filter_yield_curve-implementation-step03): Estimate the evolution of first two Nelson-Siegel parameters # Nelson-Siegel fit theta = np.zeros((t_-1, 4)) theta[0, :] = fit_nelson_siegel_yield(tau, y[0, :], par_start) for t in range(1, t_-1): theta[t, :] = fit_nelson_siegel_yield(tau, y[t, :], theta[t-1, :]) # ## [Step 4](https://www.arpm.co/lab/redirect.php?permalink=s_kalman_filter_yield_curve-implementation-step04): Estimate evolution of first two hidden factors of Kalman Filter z_KF, alpha, beta, sig2_U, alpha_z, beta_z, sig2_z = fit_state_space(dy, k_, p) x_rec = alpha + beta@z_KF[-1, :] # last recovered increment
# + times_to_maturity = np.round_(np.array([1, 2, 3, 5, 7, 8, 10]), 2) path = '../../../databases/global-databases/fixed-income/db_yields/data.csv' y_db = pd.read_csv(path, parse_dates=['dates'], skip_blank_lines=True) y = y_db[times_to_maturity.astype(float).astype(str)].values # - # ## [Step 1](https://www.arpm.co/lab/redirect.php?permalink=s_different_length_series-implementation-step01): Compute the swap rates daily changes # daily changes epsi = np.diff(y, 1, axis=0) # ## [Step 2](https://www.arpm.co/lab/redirect.php?permalink=s_different_length_series-implementation-step02): Flexible probabilities p = exp_decay_fp(len(epsi), tau_hl) # ## [Step 3](https://www.arpm.co/lab/redirect.php?permalink=s_different_length_series-implementation-step03): Maximum likelihood with flexible probabilities - complete series mu, s2 = fit_locdisp_mlfp(epsi, p=p, nu=nu, threshold=tol) # ## [Step 4](https://www.arpm.co/lab/redirect.php?permalink=s_different_length_series-implementation-step04): Drop the first portion of the observations from the 2yr and 5yr series r = int(np.floor(len(epsi) * trunc)) epsi_dl = epsi.copy() epsi_dl[:r, [1, 3]] = np.nan # drop observations # ## [Step 5](https://www.arpm.co/lab/redirect.php?permalink=s_different_length_series-implementation-step05): Maximum likelihood with flexible probabilities - different length mu_dl, s2_dl = fit_locdisp_mlfp_difflength(epsi_dl, p=p, nu=nu, threshold=tol)
x_csco = np.log(np.array(stocks.CSCO)) x_ge = np.log(np.array(stocks.GE)) # ## [Step 2](https://www.arpm.co/lab/redirect.php?permalink=s_mlfp_ellipsoid_convergence-implementation-step02): Compute the invariants using a GARCH(1,1) fit # + _, _, epsi_csco = fit_garch_fp(np.diff(x_csco)) _, _, epsi_ge = fit_garch_fp(np.diff(x_ge)) epsi = np.array([epsi_csco, epsi_ge]).T # - # ## [Step 3](https://www.arpm.co/lab/redirect.php?permalink=s_mlfp_ellipsoid_convergence-implementation-step03): Set the exp. decay probabilities for MLFP estimation and compute the effective number of scenarios p = exp_decay_fp(len(epsi_csco), tau_hl) # exp. decay flexible probabilities ens = effective_num_scenarios(p) # effective number of scenarios # ## [Step 4](https://www.arpm.co/lab/redirect.php?permalink=s_mlfp_ellipsoid_convergence-implementation-step04): Perform the MLFP estimation mu_mlfp, sig2_mlfp = fit_locdisp_mlfp(epsi, p=p, nu=nu, threshold=gamma, print_iter=True) # ## Plots # + plt.style.use('arpm')
# ## [Input parameters](https://www.arpm.co/lab/redirect.php?permalink=s_ens_exp_decay-parameters) gamma = 10 # parameter for the generalized exponential of entropy t_ = 500 # number of scenarios tau_hl_max = np.floor(1.2 * t_) # maximum half-life parameter k_ = 50 # number of half-life parameters considered # ## [Step 1](https://www.arpm.co/lab/redirect.php?permalink=s_ens_exp_decay-implementation-step01): Create a grid of half-life values for plotting tau_hl_grid = np.linspace(1, tau_hl_max, num=k_) # ## [Step 2](https://www.arpm.co/lab/redirect.php?permalink=s_ens_exp_decay-implementation-step02): Compute exponential decay probabilities p = np.zeros((k_, t_)) for k in range(k_): p[k] = exp_decay_fp(t_, tau_hl_grid[k]) # ## [Step 3](https://www.arpm.co/lab/redirect.php?permalink=s_ens_exp_decay-implementation-step03): Compute effective number of scenarios ens = np.zeros(len(tau_hl_grid)) ens_gamma = np.zeros(k_) for k in range(k_): ens[k] = effective_num_scenarios(p[k]) ens_gamma[k] = effective_num_scenarios(p[k], type_ent='gen_exp', gamma=gamma) # ## Plots # + plt.style.use('arpm')