def log_py_zM_bin_j(lambda_bin_j, y_bin_j, zM, k, nj_bin_j): ''' Compute log p(y_j | zM, s1 = k1) of the jth lambda_bin_j ( (r + 1) 1darray): Coefficients of the binomial distributions in the GLLVM layer y_bin_j (numobs 1darray): The subset containing only the binary/count variables in the dataset zM (M x r x k ndarray): M Monte Carlo copies of z for each component k1 of the mixture k (int): The number of components of the mixture nj_bin_j (int): The number of possible values/maximum values of the jth binary/count variable -------------------------------------------------------------- returns (ndarray): p(y_j | zM, s1 = k1) ''' M = zM.shape[0] r = zM.shape[1] numobs = len(y_bin_j) yg = np.repeat(y_bin_j[np.newaxis], axis = 0, repeats = M) yg = yg.astype(np.float) nj_bin_j = np.float(nj_bin_j) coeff_binom = binom(nj_bin_j, yg).reshape(M, 1, numobs) eta = np.transpose(zM, (0, 2, 1)) @ lambda_bin_j[1:].reshape(1, r, 1) eta = eta + lambda_bin_j[0].reshape(1, 1, 1) # Add the constant den = nj_bin_j * log_1plusexp(eta) num = eta @ y_bin_j[np.newaxis, np.newaxis] log_p_y_z = num - den + np.log(coeff_binom) return np.transpose(log_p_y_z, (0, 2, 1)).astype(np.float)
def mvt_logpdf(x, mu, Li, df): dim = Li.shape[0] Ki = np.dot(Li.T, Li) #determinant is just multiplication of diagonal elements of cholesky logdet = 2*log(1./np.diag(Li)).sum() lpdf_const = (gammaln((df + dim) / 2) -(gammaln(df/2) + (log(df)+log(np.pi)) * dim*0.5 + logdet * 0.5) ) x = np.atleast_2d(x) if x.shape[1] != mu.size: x = x.T assert(x.shape[1] == mu.size or x.shape[0] == mu.size) d = (x - mu.reshape((1 ,mu.size))).T Ki_d_scal = np.dot(Ki, d) /df #vector d_Ki_d_scal_1 = diag_dot(d.T, Ki_d_scal) + 1. #scalar res_pdf = (lpdf_const - 0.5 * (df + dim) * np.log(d_Ki_d_scal_1)).flatten() if res_pdf.size == 1: res_pdf = np.float(res_pdf) return res_pdf
def _logL(self, beta, X, y): """The log likelihood.""" n_samples = np.float(X.shape[0]) l = np.dot(X, beta) #logL = -0.5 * 1. / n_samples * np.sum((y - l)**2) logL = -0.5 * np.sum((y - l)**2) return logL
def _grad_L2loss(self, beta, reg_lambda, X, y): n_samples = np.float(X.shape[0]) z = np.dot(X, beta) #grad_beta = 1. / n_samples * np.transpose(np.dot(np.transpose(z - y), X)) grad_beta = np.transpose(np.dot(np.transpose(z - y), X)) print('grad_beta 0,1', grad_beta[0:2]) return grad_beta
def get_learning_rate_exp_decay(global_step, base_lr=0.01, decay_rate=0.9, decay_steps=2, staircase=True): if staircase: exponent = global_step / decay_steps # integer division else: exponent = global_step / np.float(decay_steps) return base_lr * np.power(decay_rate, exponent)
def _grad_L2loss(self, beta, X, y): #print(beta.shape,X.shape,y.shape) if y.ndim == 1: y = y[:, np.newaxis] n_samples = np.float(X.shape[0]) z = np.dot(X, beta) #grad_beta = 1. / n_samples * np.transpose(np.dot(np.transpose(z - y), X)) grad_beta = np.transpose(np.dot(np.transpose(z - y), X)) #print('gb',grad_beta.shape) return grad_beta
def chebyshev_centre(A, b, gamma): rows, cols = A.shape c = np.zeros(cols + 1) c[-1] = -1 A_ = np.hstack([A, np.sqrt(np.sum(np.power(A, 2), axis=1)).reshape(-1, 1)]) A_ = np.vstack([A_, -c.reshape(1, -1)]) b_ = np.append(b, 100).reshape(-1, 1) # l2 norm minimisation of w P = gamma * np.eye(cols + 1) P[:, -1] = P[-1, :] = 0 res = solve_qp(P=P, q=c, G=A_, h=b_) x_c = np.array(res[:-1]) R = np.float(res[-1]) return x_c, R
def __init__(self, mu, K, Ki = None, logdet_K = None, L = None): mu = np.atleast_1d(mu).flatten() K = np.atleast_2d(K) assert(np.prod(mu.shape) == K.shape[0] ) assert(K.shape[0] == K.shape[1]) self.mu = mu self.K = K (val, vec) = np.linalg.eigh(K) idx = np.arange(mu.size-1,-1,-1) (self.eigval, self.eigvec) = (np.diag(val[idx]), vec[:,idx]) self.eig = self.eigvec.dot(np.sqrt(self.eigval)) self.dim = K.shape[0] #(self.Ki, self.logdet) = (np.linalg.inv(K), np.linalg.slogdet(K)[1]) (self.Ki, self.L, self.Li, self.logdet) = pdinv(K) self.lpdf_const = -0.5 *np.float(self.dim * np.log(2 * np.pi) + self.logdet)
def run_cavi(self, tau, nu, phi_mu, phi_var, max_iter=200, tol=1e-6): params = packing.flatten_params(tau, nu, phi_mu, phi_var) self.trace.reset() diff = np.float('inf') while diff > tol and self.trace.stepnum < max_iter: self.cavi_updates(tau, nu, phi_mu, phi_var) new_params = packing.flatten_params(tau, nu, phi_mu, phi_var) diff = np.max(np.abs(new_params - params)) self.trace.update(params, diff) if not np.isfinite(diff): print('Error: non-finite parameter difference.') break params = new_params if self.trace.stepnum >= max_iter: print('Warning: CAVI reached max_iter.') print('Done with CAVI.') return tau, nu, phi_mu, phi_var
def __init__(self, mu, K, df, Ki = None, logdet_K = None, L = None): mu = np.atleast_1d(mu).flatten() K = np.atleast_2d(K) assert(np.prod(mu.shape) == K.shape[0] ) assert(K.shape[0] == K.shape[1]) self.mu = mu self.K = K self.df = df self._freeze_chi2 = stats.chi2(df) self.dim = K.shape[0] self._df_dim = self.df + self.dim #(self.Ki, self.logdet) = (np.linalg.inv(K), np.linalg.slogdet(K)[1]) (self.Ki, self.L, self.Li, self.logdet) = pdinv(K) self.lpdf_const = np.float(gammaln((self.df + self.dim) / 2) -(gammaln(self.df/2) + (log(self.df)+log(np.pi)) * self.dim*0.5 + self.logdet * 0.5) )
def config_to_str(config): batch_size_frac = config['batch_size'] / np.float(config['N']) cstr = 'B{:0.2f}-L{:0.2f}-M{}'.format(batch_size_frac, config['lr_decay'], config['momentum']) return cstr
def forward(self, x=None): return np.float(1)
def fit(self): group = self.group lambdas = self.reg_lambda X = self.xs y = self.ys np.random.RandomState(0) group = np.array(group) group.dtype = np.int64 if group.shape[0] != X.shape[1]: raise ValueError('group should be (n_features,)') # type check for data matrix if not isinstance(X, np.ndarray): raise ValueError('Input data should be of type ndarray (got %s).' % type(X)) n_features = np.float(X.shape[1]) n_features = np.int64(n_features) if y.ndim == 1: y = y[:, np.newaxis] n_classes = 1 n_classes = np.int64(n_classes) # Initialize parameters beta_hat = 1 / (n_features) * np.random.normal(0.0, 1.0, [n_features, n_classes]) #print('betahat 0,1', beta_hat[0:2]) fit_params = list() for l, rl in enumerate(lambdas): #print(l, rl) fit_params.append({'beta': beta_hat}) # Warm initialize parameters if l == 0: fit_params[-1]['beta'] = beta_hat else: fit_params[-1]['beta'] = fit_params[-2]['beta'] tol = self.tol alpha = 1. # Temporary parameters to update beta = np.zeros([n_features, n_classes]) beta = fit_params[-1]['beta'] g = np.zeros([n_features, n_classes]) # Initialize loss accumulators L, DL = list(), list() #pdb.set_trace() for t in range(0, self.max_iter): #print(t) grad_beta = self._grad_L2loss(beta=beta, reg_lambda=rl, X=X, y=y) #print('before grad',beta[0:2]) beta = beta - self.learning_rate * grad_beta #beta = beta - self.learning_rate * (1 / np.sqrt(t + 1)) * g #print('after grad',beta[0:2]) beta = self._prox(beta, rl * self.learning_rate) #print('after prox',beta[0:2]) #print(beta[0]) L.append(self._loss(beta, rl, X, y)) #print(self._loss(beta, rl, X, y)) if t > 1: if (L[-1] > L[-2]): break DL.append(L[-1] - L[-2]) #print('DL, L-1, L-2',DL, L[-1], L[-2]) if np.abs(DL[-1] / L[-1]) < tol: print('converged', rl) msg = ('\tConverged. Loss function:' ' {0:.2f}').format(L[-1]) msg = ('\tdL/L: {0:.6f}\n'.format(DL[-1] / L[-1])) break #print(beta) fit_params[-1]['beta'] = beta self.lossresults[rl] = L self.dls[rl] = DL #print(L) # Update the estimated variables self.fit_ = fit_params self.ynull_ = np.mean(y) # Return return self
def get_learning_rate_exp_decay(global_step, base_lr=0.01, decay_rate=0.9, decay_steps=2, staircase=True): if staircase: exponent = global_step / decay_steps # integer division else: exponent = global_step / np.float(decay_steps) return base_lr * np.power(decay_rate, exponent)
def dUdz(z,e): return (pDist(z+e)-pDist(z))/e smpls=50000; burnin=100000; total=smpls+burnin; z = np.empty(total); z[0] = 0; r = np.empty(total); r[0] = 0; accept = np.zeros(total); ratio = np.zeros(total); unif_RV = np.random.uniform(size=total) eps = np.array([0.005,0.01,0.1,0.2,0.5, 1]); e = 5; eps1 = eps[e]; L = 10; # leapgrog steps M = np.array([1]); s = np.float(M[0]) # mass definition matrix #accRatio = np.zeros((len(eps))) for i in np.arange(0,total-1): r[i] = np.random.normal(0,s) Kr = (1/2)*r[i]**2*(s**(-1)) # kinetic energy Uz = -np.log(pDist(z[i])) # potential energ H = Uz + Kr # total energy for steps in range(L): r_half = r[i] - (eps1/2)*dUdz(z[i], eps1); z_whole = z[i] + eps1*(r_half/s); r_whole = r_half - (eps1/2)*dUdz(z_whole, eps1);
def config_to_str(config): batch_size_frac = config['batch_size'] / np.float(config['N']) cstr = 'B{:0.2f}-L{:0.2f}-M{}'.format(batch_size_frac, config['lr_decay'], config['momentum']) return cstr
def log_pdf(self, y, x=None, node=None): self.cur_log_pdf = self.compute(log_pdf, y, x, node) if len(np.array(self.cur_log_pdf).shape) == 0: self.cur_log_pdf = np.float(self.cur_log_pdf) return self.cur_log_pdf
cols = (repeat(asarray(Blf.col).reshape(Blf.nnz, 1) * M, M, 1) + ks).ravel() vals = (repeat(asarray(Blf.data).reshape(Blf.nnz, 1), M, 1)).ravel() newBlf = scipy.sparse.coo_matrix( (vals, (rows, cols)), shape=(shape[0] * M, shape[1] * M)).tocsr() return newBlf if __name__ == "__main__": global img_file img_file = sys.argv[1] KS_file_name = sys.argv[2] Thickness_file_name = sys.argv[3] output_prefix = sys.argv[4] W_w = np.float(sys.argv[5]) W_sparse = np.float(sys.argv[6]) print 'W_sparse', W_sparse solve_choice = np.int(sys.argv[7]) W_spatial = np.float(sys.argv[8]) print 'W_spatial', W_spatial global save_for_application_path_prefix save_for_application_path_prefix = "./Application_Files/" W_neighbors = 0.0 if solve_choice == 3 or solve_choice == 6: #### solve per pixel with neighborhood info constraints W_neighbors = np.float(sys.argv[9]) print 'W_neighbors', W_neighbors
def fit(self): group = self.group print(group.shape) lambdas = self.reg_lambda parameter = self.parameter X = self.xs #print(X.shape) y = self.ys np.random.RandomState(0) group = np.asarray(group, dtype=np.int64) #print(group.shape[0]) group.dtype = np.int64 #print(group.shape[0]) #print(X.shape[1]) if group.shape[0] != X.shape[1]: raise ValueError('group should be (n_features,)') # type check for data matrix if not isinstance(X, np.ndarray): raise ValueError('Input data should be of type ndarray (got %s).' % type(X)) n_features = np.float(X.shape[1]) n_features = np.int64(n_features) if y.ndim == 1: y = y[:, np.newaxis] self.ys = y #print(y.shape) n_classes = 1 n_classes = np.int64(n_classes) beta_hat = 1 / (n_features) * np.random.normal(0.0, 1.0, [n_features, n_classes]) fit_params = list() for l, rl in enumerate(lambdas): fit_params.append({'beta': beta_hat}) if l == 0: fit_params[-1]['beta'] = beta_hat else: fit_params[-1]['beta'] = fit_params[-2]['beta'] tol = self.tol alpha = 1. beta = np.zeros([n_features, n_classes]) beta = fit_params[-1]['beta'] #print('losser',self._L2loss(beta,X,y)) g = np.zeros([n_features, n_classes]) L, DL, L2, PEN = list(), list(), list(), list() lamb = self.learning_rate bm1 = beta.copy() bm2 = beta.copy() for t in range(0, self.max_iter): L.append(self._loss(beta, rl, X, y)) L2.append(self._L2loss(beta, X, y)) PEN.append(self._L1penalty(beta)) w = (t / (t + 3)) yk = beta + w * (bm1 - bm2) #print('losser',self._L2loss(yk,X,y)) #print('beforebt',np.linalg.norm(yk),np.linalg.norm(X),np.linalg.norm(y)) beta, lamb = self._btalgorithm(yk, lamb, .5, 1000, rl) #X = self.xs #y = self.ys #print('losser2',self._L2loss(beta,X,y)) bm2 = bm1.copy() bm1 = beta.copy() if t > 1: DL.append(L[-1] - L[-2]) if np.abs(DL[-1] / L[-1]) < tol: print('converged', rl) msg = ('\tConverged. Loss function:' ' {0:.2f}').format(L[-1]) msg = ('\tdL/L: {0:.6f}\n'.format(DL[-1] / L[-1])) break #print(beta) fit_params[-1]['beta'] = beta self.lossresults[rl] = L self.l2loss[rl] = L2 self.penalty[rl] = PEN self.dls[rl] = DL #print(L) # Update the estimated variables self.fit_ = fit_params self.ynull_ = np.mean(y) # Return return self