def full_conditional_Beta(y, X, sigma_iter, eta, inv_C): inv_Vb = inv_C + (matfuncs.dot(X.T,X))/sigma_iter Vb = np.linalg.inv(inv_Vb) X_tild = matfuncs.dot(inv_C, eta) + (matfuncs.dot(X.T, y))/ sigma_iter mub = matfuncs.dot(Vb, X_tild) Beta_iter = RanMNormalPrec(mub, inv_Vb) return(Beta_iter)
def full_conditional_bu(num_zeros, num_splines, Knots, Z, Zt, sigma_e_iter, sigma_u_iter, y_tild): # prior matrix over linear and non linear effects # the linear effects get the prior 1/100, the splines get the # penalizing prior 1/sigma_u2 D = get_D(num_zeros, num_splines, 1 , sigma_u_iter, Knots) inv_V_bu = (D + matfuncs.dot(Zt,Z) / sigma_e_iter) V_bu = np.linalg.inv(inv_V_bu) mu_bu = matfuncs.dot(V_bu, matfuncs.dot(Zt,y_tild) / sigma_e_iter) bu = RanMNormalPrec(mu_bu, inv_V_bu) return(bu)
def update_topics(Kmax, k_tild, K, U1, n_m_z, n_z_t, n_z, z_m_n, tau, m, n, t, gamma, alpha, Beta_iter, Beta_new, BtM): # In order for the number of topics not to explode before convergence if k_tild >= Kmax: k_tild = Kmax-1 if k_tild < int(K): # Preserve the topic in the used topics active = which(U1, 1) k_star = active[int(k_tild)] # and increase the counters n_m_z, n_z_t, n_z, z_m_n = increment(int(k_star), n_m_z, n_z_t, n_z, z_m_n, int(m), int(n), int(t)) BtM[int(m)] += Beta_iter[int(k_star)] else: deactive = which(U1, 0) k_star = deactive[0] U1[int(k_star)] = 1. Beta_iter[int(k_star)] = Beta_new Beta_new = np.random.normal(0., 1.) # Incerement the counters n_m_z, n_z_t, n_z, z_m_n = increment(int(k_star), n_m_z, n_z_t, n_z, z_m_n, int(m), int(n), int(t)) K += 1 tau = local_sample_tau(tau, gamma, K) # update scalar product BtM = matfuncs.dot(n_m_z, Beta_iter.T) return(K, U1, n_m_z, n_z_t, n_z, z_m_n, tau, Beta_iter, Beta_new, BtM)
def plot_spline_pure_effect(j): xL_ = np.zeros((100, xL.shape[1])) xS_ = get_x_range(xS) for k in range(xS_.shape[1]): if k != j: xS_[:,k] = 0 X = get_X(xL_, xS_, p) Z = get_Z(xS_, Knots, p) for t in range( np.max(time) + 1): yhat1 = matfuncs.dot(X, thetaH[:,t,:].T) yhat2 = matfuncs.dot(Z, UH.T) yhat = yhat1 + yhat2 yhat = yhat - np.mean(yhat) sns.tsplot(data=yhat.T,ci=[95]) plt.axhline(y = 0, color='black', linestyle='--')
def full_conditional_sigma(Y, X, theta, T, time): a, b = 1., 1. N = len(Y) a1 = a + float(N)/2 b1 = 0. for t in range(1,int(T)): idxt = which(time, t) Yt = sub_array(Y, idxt) Xt = sub_row_matrix(X, idxt) b1 += np.sum(np.power(Yt - matfuncs.dot(Xt, theta[t]), 2)) b1 = 1/ (b1 /2. + 1/b) sigma_iter = 1/np.random.gamma(a1, b1, 1) return(sigma_iter[0])
def plot_spline_pure_effect(j): xL_ = np.zeros((100, xL.shape[1])) xS_ = get_x_range(xS) for k in range(xS_.shape[1]): if k != j: xS_[:,k] = 0 X = get_X(xL_, xS_, p) Z = get_Z(xS_, Knots, p) ax = plt.subplot(projection='3d') for t in range(1, np.max(time) + 1): yhat1 = matfuncs.dot(X, thetaH[:,t,:].T) yhat2 = matfuncs.dot(Z, UH.T) yhat = yhat1 + yhat2 yhat = yhat - np.mean(yhat) surf = np.mean(yhat, axis =1 ) ax.plot(xS_[:,j], np.ones(100) * t ,surf, color='b') #ax.add_collection3d(pl.fill_between(x, 0.95*z, 1.05*z, color='r', alpha=0.3), zs=t, zdir='y') ax.set_ylabel('Decade') ax.set_zlabel('Success')
def gibbs_dlm_1iter(X, Y, time, theta, sigmae, W): T = np.max(time) + 1 K = X.shape[1] #Forward filetring at = np.zeros((T,K)) Rt = np.zeros((T,K,K)) mt = np.zeros((T,K)) Ct = np.zeros((T,K,K)) # Initial state at[0] = np.zeros(K) Rt[0] = np.diag(np.ones(K)) mt[0] = np.zeros(K) # prior over the mean of the thetas Ct[0] = 100 * np.diag(np.ones(K)) # prior over the variance of the thetas for t in range(1,T): idxt = which(time, t) Xt = sub_row_matrix(X, idxt) Yt = sub_array(Y, idxt) at[t] = mt[t-1] Rt[t] = Ct[t-1] + W inv_Rt = np.linalg.inv(Rt[t]) inv_Ct = inv_Rt + matfuncs.dot(Xt.T, Xt) / sigmae Ct[t] = np.linalg.inv(inv_Ct) mt[t] = matfuncs.dot(Ct[t], matfuncs.dot(inv_Rt, at[t]) + matfuncs.dot(Xt.T, Yt) / sigmae) # Backward smoothing ht = np.zeros((T,K)) Ht = np.zeros((T,K,K)) theta[T-1] = RanMNormalPrec(mt[int(T-1)], np.linalg.inv(Ct[int(T-1)])) for t in range(T-2,-1, -1): Ht[t] = np.linalg.inv(np.linalg.inv(Ct[t]) + np.linalg.inv(W)) ht[t] = matfuncs.dot(Ht[t], matfuncs.dot(np.linalg.inv(Ct[t]), mt[t]) + matfuncs.dot(np.linalg.inv(W), theta[t+1])) theta[t] = RanMNormalPrec(ht[t], np.linalg.inv(Ht[t])) sigmae = full_conditional_sigma(Y, X, theta, T, time) w = np.zeros(int(K)) for k in range(K): thetat1 = theta[1:,k] thetat = theta[:int(T-1),k] w[k] = full_conditional_W(thetat1, thetat) W = np.diag(w) return(theta, sigmae, W)
def DSPsHDP(Iter, y, time, data, max_len_doc, num_docs, V, xL, xS, Knots, p): """ Non prametric Regression """ """ Dynamic part """ maxT = np.max(time) + 1 """ Spline part """ # We allow for an intercept here # this is the linear part of the matrix X = get_X(xL, xS, p) # this accounts for the fluctuation in the splines Z = get_Z(xS, Knots, p) Zt = Z.T # we are spliting the estimation in two parts # thus we require the number of the lienar part (0 on the diagonal) # to be exactly 0 num_zeros = 0 num_splines = xS.shape[1] """ Initializing parameters """ ### Dynamic part theta = np.zeros((maxT,X.shape[1])) sigmae = 1 W = 1 * np.diag(np.ones(X.shape[1])) ### Spline part sigma_u_iter = np.ones(int(num_splines)) * 1000 #np.random.random(num_splines) U_iter = np.random.random(Z.shape[1]) """ sHDP """ # Priors fixed alpha = 0.01 beta = 0.50 # Dispersion of topics gamma = 0.01 # Initial number of themes K0, maximum number of themes is Kmax K0 = 30 Kmax = 200 K = K0 # Init the state of the sample: dynamic/static Fixed_K = False # Lists of used (1) and unused topics (0) U1 = np.zeros(int(Kmax)) # Activated topics U1[:int(K)] = 1. # Init n_m_z, n_z_t, n_z, z_m_n = Init(data, K0, Kmax, V, num_docs, max_len_doc) n_m = init_nm(n_m_z) Zbar = get_active_Zbar(n_m_z , n_m, U1, K) # Initialization of dirichlet paramters tau tau = np.ones(int(K)) / np.float(K) T,tau = sample_tau(tau, gamma, alpha, n_m_z, K, U1) gamma = sample_gamma(T, gamma, int(K)) alpha = sample_alpha(T, alpha, n_m) # Regression history Beta_iter = np.zeros(int(Kmax)) Beta_iter[int(K):] = 0 Beta_iter_tmp = np.zeros(int(K)) Beta_new = np.random.normal(0, 100.) """ History and caching """ # History for saving the results Hist = np.zeros(int(Iter)) """ Start the Markov chain """ for iteration in range(Iter): """ Sample Managerial, Acoustical and Sonor effects """ # sample spline yhat0 = matfuncs.dot(Zbar, Beta_iter_tmp) yhat1 = matfuncs.dot(X, theta.T) y_tild1 = np.zeros(Y.shape) for t in range(Y.shape[0]): y_tild1[t] = Y[t] - yhat1[t, time[t]] - yhat0[t] U_iter, sigma_u_iter = SP_Reg_iter(num_zeros, num_splines, Knots, Z, Zt, sigmae, sigma_u_iter, y_tild1, theta) # sample SSM yhat2 = matfuncs.dot(Z, U_iter) y_tild2 = Y - yhat2 - yhat0 theta, sigmae, W = gibbs_dlm_1iter(X, y_tild2, time, theta, sigmae, W) """ Themal effects """ # Themes effects before sampling BtM = matfuncs.dot(n_m_z, Beta_iter) # Keep what is not explained by the acoustics yres = np.zeros(Y.shape[0]) for t in range(Y.shape[0]): yres[t] = Y[t] - (yhat1[t, time[t]] + yhat2[t]) for i in range(len(data)): m = int(data[i][0]) n = int(data[i][1]) t = int(data[i][2]) # Discount for n-th word t with topic z z = z_m_n[m][n] n_m_z, n_z_t, n_z, z_m_n = decrement(int(z), n_m_z, n_z_t, n_z, z_m_n, int(m), int(n), int(t)) # Number of words in the document and the BtZ of the document Nd = np.int(n_m[m]) yd = yres[m] # Update scalar product BtM[m] -= Beta_iter[int(z)] BtMd = BtM[m] # Tockenrobabilties phdp = get_prob(U1, n_z_t, n_m_z, n_z, alpha, beta, tau, K, V,int(m), int(t)) active = which(U1, 1.) prob = get_pshdp(phdp, BtMd, K, yd, Beta_iter, Beta_new, sigmae, Nd, active) if Fixed_K == True: # Do not allow for sampling new topics prob[int(K)] = 0. k_tild = multinomial_rvs(prob) K, U1, n_m_z, n_z_t, n_z, z_m_n, tau, Beta_iter, Beta_new, BtM = update_topics(int(Kmax), int(k_tild), int(K), U1, n_m_z, n_z_t, n_z, z_m_n, tau, int(m), int(n), int(t), gamma, alpha, Beta_iter, Beta_new, BtM) # Check the topics still activated del_idx = which(n_z, 0.) if del_idx[0] >= 0.: for k in del_idx: U1[int(k)] = 0. Beta_iter[int(k)] = 0. K = np.sum(U1) """ Update HDP priors """ # Update the priors T,tau = sample_tau(tau, gamma, alpha, n_m_z, int(K), U1) gamma = sample_gamma(T, gamma, int(K)) alpha = sample_alpha(T, alpha, n_m) """ Sample Thematic parameters """ Zbar = get_active_Zbar(n_m_z , n_m, U1, K) # The priors are dynamic and adjust to the number of topics that are sampled eta = np.zeros(int(K)) inv_C = np.diag(np.ones(int(K))) * 0.01 Beta_iter_tmp = full_conditional_Beta(yres, Zbar, sigmae, eta, inv_C) # Relate the coefficients to the active topics and update the new topics sample active = which(U1, 1.) for t, k in enumerate(active): Beta_iter[int(k)] = Beta_iter_tmp[int(t)] Beta_new = np.random.normal(0, 100.) """ Checking convergence and caching the results """ # Once a reasonable number of iterations is reached # we fix the topics and start sampling if iteration >= int(0.9 * Iter): if Fixed_K == False: # Guess which number of topics was sampled the most # in the last 90% iterations if Iter > 5: # loc the best number of topics as soon as the counter gets back to it best_K = find_max_topic(Hist[:iteration], 0.75) if K == best_K: iter_left = Iter - iteration Fixed_K = True # Update the priors accordingly T,tau = sample_tau(tau, gamma, alpha, n_m_z, int(K), U1) # Updating the hyperpriors gamma = sample_gamma(T, gamma, int(K)) alpha = sample_alpha(T, alpha, n_m) # Init Gibbs samples history SIGMA_E = np.zeros( np.int64(Iter - iteration)) SIGMA_U = np.zeros((np.int64(Iter - iteration), np.int64(num_splines))) #BU = np.zeros((np.int64(Iter - iteration), np.int64(C.shape[1]))) iter_hist = 0 # Init phi and theta phi = np.zeros((np.int64(K), np.int64(V))) theta_topic = np.zeros((np.int64(num_docs), np.int64(K))) # Regression results B = np.zeros((np.int64(Iter - iteration), np.int64(Kmax))) # The splines thetaH = np.zeros((np.int64(Iter - iteration), np.int64(maxT), X.shape[1])) WH = np.zeros((np.int64(Iter - iteration), X.shape[1], X.shape[1])) UH = np.zeros((np.int64(Iter - iteration), Z.shape[1])) if Fixed_K == True: # Fix the theta prior fixed_alpha = alpha * tau[:-1] # The estimated phi/theta active = which(U1, 1.) phi += Compute_phi(K, V, n_z_t, beta, active) / float(iter_left) theta_topic += Compute_theta(num_docs, K, n_m_z, active, fixed_alpha) / float(iter_left) #S += sigma_iter/ float(iter_left) B[int(iter_hist)] = Beta_iter #BU[int(iter_hist)] = BetaU_iter SIGMA_E[int(iter_hist)] = sigmae SIGMA_U[int(iter_hist)] = sigma_u_iter # saving the history of the SS plines thetaH[int(iter_hist)] = theta #sigmae_H[iteration] = sigmae #sigmau_H[iteration] = sigma_u_iter WH[int(iter_hist)] = W UH[int(iter_hist)] = U_iter iter_hist += 1. Hist[iteration] = K return(Hist, phi, theta_topic, B, active, thetaH, WH, UH, SIGMA_E, SIGMA_U)
def full_conditional_sigma_e(y, X, Z, b, u): ae, be = 1., 1. a1 = ae + len(y) * 0.5 b1 = 1/(be + 0.5 * np.sum(np.square(y - matfuncs.dot(X, b) - matfuncs.dot(Z, u)))) sigma_e_iter = 1/np.random.gamma(a1, b1, 1) return(sigma_e_iter)
def RanMNormalPrec(mub, inv_Vb): c = np.linalg.cholesky(inv_Vb) rv = np.random.normal(loc= 0, scale= 1, size= len(mub)) draw = mub + matfuncs.dot(np.linalg.inv(c.T), rv) return(draw)
x1 = np.random.uniform(0, 5, 1)[:,None] x2 = np.random.uniform(0, 5, 1)[:,None] # Non linear part x3 = np.random.uniform(0, 5, 1)[:,None] x4 = np.random.uniform(0, 5, 1)[:,None] # error term e = np.random.normal(0, sigmae, 1)[:,None] # dependent variable topic_idx = np.random.choice(3,1)[0] data_text.append(rawdocs[topic_idx]) #y = x1 + x2 + 3 * x3 *np.cos((x3 - 3)*5) + 3*x4*np.cos(x4*5) + e #y = y * np.power(-1, t) x = np.concatenate((x1, x2, x3, x4), axis =1) y = ytopics[topic_idx] + matfuncs.dot(x,true_theta[t]) + np.random.normal(0, sigmae, 1)[0] # The linear effects xl = np.concatenate((x1, x2), axis =1) # The non linear effects xs = np.concatenate((x3, x4), axis =1) # Saving the variables xL.extend(xl) xS.extend(xs) X.extend(x) Y.extend(y) time.append(t) #X = np.matrix(X) xL = np.matrix(xL) xS = np.matrix(xS)