def main(): Log.set_loglevel(logging.DEBUG) prior = Gaussian(Sigma=eye(2) * 100) posterior = OzonePosterior(prior, logdet_alg="scikits", solve_method="scikits") proposal_cov = diag([ 4.000000000000000e-05, 1.072091680000000e+02]) mcmc_sampler = StandardMetropolis(posterior, scale=1.0, cov=proposal_cov) start = asarray([-11.35, -13.1]) mcmc_params = MCMCParams(start=start, num_iterations=5000) chain = MCMCChain(mcmc_sampler, mcmc_params) chain.append_mcmc_output(StatisticsOutput(print_from=1, lag=1)) home = expanduser("~") folder = os.sep.join([home, "sample_ozone_posterior_average_serial"]) store_chain_output = StoreChainOutput(folder) chain.append_mcmc_output(store_chain_output) loaded = store_chain_output.load_last_stored_chain() if loaded is None: logging.info("Running chain from scratch") else: logging.info("Running chain from iteration %d" % loaded.iteration) chain = loaded chain.run() f = open(folder + os.sep + "final_chain", "w") dump(chain, f) f.close()
def low_rank_approx(K, d): """ Returns a low rank approximation factor L of the given psd matrix such that LL^T \approx K with a given number of principal components to use K - psd matrix to compute low-rank approximation of d - number of principal components to use returns (L, s, V) where L - LL^T \approx K s - 1D vector of Eigenvalues V - matrix containing Eigen-row-vectors """ # perform SVD and only use first d components. Note that U^T=V if K psd and # rows of V are Eigenvectors of K U, s, V = svd(K) U = U[:, 0:d] V = V[0:d, :] s = s[0:d] S = diag(s) # K \approx=U.dot(S.dot(V)) L = sqrt(S).dot(V) # LL^T \approx K return (L, s, V)
def fit(self, func, params_dist, pre_X = None, pre_y = None): time_start = time.time() if self.random_state is not None: np.random.seed(self.random_state) grid, grid_scaled = self.get_grid(params_dist) mu = np.zeros(self.n_grid) + self.mu_prior sigma = np.ones(self.n_grid)*self.sigma_prior X = np.zeros((self.max_iter, self.n_params)) X_scaled = np.matrix(np.zeros((self.max_iter, self.n_params))) y = np.zeros(self.max_iter) if (pre_X is not None) and (pre_y is not None): pre_X_mat, pre_X_scaled = self.scale(pre_X, pre_y, params_dist) X = np.vstack([pre_X_mat, X]) X_scaled = np.vstack([pre_X_scaled, X_scaled]) y = np.concatenate([pre_y, y]) pre_len = len(pre_y) else: pre_len = 0 if self.verbose: params_name = [i[:9] for i in self.params_name] logger.info('%4s|%9s|%9s|%9s', 'Iter','Func','Max', '|'.join(['{:9s}'.format(i) for i in params_name])) for i in xrange(pre_len, pre_len + self.max_iter): #beta = (i + 1)**2 if self.beta_mode == 'log': d = len(self.params_name) beta = 2*np.log(2 *(i + 1)**2 * np.pi**2 /.3) + \ 2*d*np.log( (i+1)**2 * d * 2) elif self.beta_mode == 'linear': beta = i + 1 elif self.beta_mode == 'square': beta = (i + 1)**2 else: logger.error("What The Hell. Change Beta Parameter") idx = np.argmax(mu + np.sqrt(beta)*sigma) X[i,:] = grid[idx] X_scaled[i] = grid_scaled[idx] y[i] = func(**dict(zip(self.params_name, X[i]))) KT = self.kernel(X_scaled[:(i + 1)], X_scaled[:(i + 1)])*\ self.sigma_prior invKT = inv(KT + self.sig**2*identity(i + 1)) grid, grid_scaled = self.get_grid(params_dist) kT = self.kernel(X_scaled[:(i + 1)], grid_scaled)*\ self.sigma_prior**2 mu = self.mu_prior + \ kT.T.dot(invKT).dot(y[:(i + 1)] - self.mu_prior) sigma2 = np.ones(self.n_grid)*self.sigma_prior**2 - \ diag(kT.T.dot(invKT).dot(kT)) sigma = np.sqrt(sigma2) ### Save Data if self.verbose: logger.info('%4d|%9.4g|%9.4g|%s', i, y[i], np.max(y[:(i + 1)]), '|'.join(['{:9.4g}'.format(ii) for ii in X[i]])) if time.time() - time_start > self.time_budget: break self.X = X[:(i + 1)] self.y = y[:(i + 1)] self.mu = mu self.beta = beta self.sigma = sigma self.grid = grid
def corrcoef(x, y=None, rowvar=1, bias=0): """The correlation coefficients """ c = cov(x, y, rowvar, bias) try: d = diag(c) except ValueError: # scalar covariance return 1 return c/sqrt(multiply.outer(d,d))
def main(): Log.set_loglevel(logging.DEBUG) prior = Gaussian(Sigma=eye(2) * 100) num_estimates = 1000 home = expanduser("~") folder = os.sep.join([home, "sample_ozone_posterior_rr_sge"]) # cluster admin set project jump for me to exclusively allocate nodes parameter_prefix = "" # #$ -P jump" cluster_parameters = BatchClusterParameters(foldername=folder, memory=7.8, loglevel=logging.DEBUG, parameter_prefix=parameter_prefix, max_walltime=60 * 60 * 24 - 1) computation_engine = SGEComputationEngine(cluster_parameters, check_interval=10) rr_instance = RussianRoulette(1e-3, block_size=400) posterior = OzonePosteriorRREngine(rr_instance=rr_instance, computation_engine=computation_engine, num_estimates=num_estimates, prior=prior) posterior.logdet_method = "shogun_estimate" proposal_cov = diag([ 4.000000000000000e-05, 1.072091680000000e+02]) mcmc_sampler = StandardMetropolis(posterior, scale=1.0, cov=proposal_cov) start = asarray([-11.55, -10.1]) mcmc_params = MCMCParams(start=start, num_iterations=5000) chain = MCMCChain(mcmc_sampler, mcmc_params) # chain.append_mcmc_output(PlottingOutput(None, plot_from=1, lag=1)) chain.append_mcmc_output(StatisticsOutput(print_from=1, lag=1)) store_chain_output = StoreChainOutput(folder, lag=1) chain.append_mcmc_output(store_chain_output) loaded = store_chain_output.load_last_stored_chain() if loaded is None: logging.info("Running chain from scratch") else: logging.info("Running chain from iteration %d" % loaded.iteration) chain = loaded chain.run() f = open(folder + os.sep + "final_chain", "w") dump(chain, f) f.close()
def log_pdf_at_quantile(self, alphas): """ Computes the log-pdf at a given 1d-vector of quantiles """ chi2_instance = chi2(self.dimension) cuttoffs = chi2_instance.isf(1 - alphas) log_determinant_part = -sum(log(diag(self.L))) quadratic_part = -0.5 * cuttoffs const_part = -0.5 * len(self.L) * log(2 * pi) return const_part + log_determinant_part + quadratic_part
def calc_error(xi, sef): Xk = self._get_X(xi).T covarM = matrix(self.var_covar) varY_hat = (Xk.T * covarM * Xk) varY_hat = sum(diag(varY_hat)) if error_calc == 'sem': se = sef * sqrt(varY_hat) else: se = sqrt(sef ** 2 + sef ** 2 * varY_hat) return se
def main(): Log.set_loglevel(logging.DEBUG) modulename = "sample_ozone_posterior_average_slurm" if not FileSystem.cmd_exists("sbatch"): engine = SerialComputationEngine() else: johns_slurm_hack = "#SBATCH --partition=intel-ivy,wrkstn,compute" johns_slurm_hack = "#SBATCH --partition=intel-ivy,compute" folder = os.sep + os.sep.join(["nfs", "data3", "ucabhst", modulename]) batch_parameters = BatchClusterParameters(foldername=folder, max_walltime=24 * 60 * 60, resubmit_on_timeout=False, memory=3, parameter_prefix=johns_slurm_hack) engine = SlurmComputationEngine(batch_parameters, check_interval=1, do_clean_up=True) prior = Gaussian(Sigma=eye(2) * 100) num_estimates = 100 posterior = OzonePosteriorAverageEngine(computation_engine=engine, num_estimates=num_estimates, prior=prior) posterior.logdet_method = "shogun_estimate" proposal_cov = diag([ 4.000000000000000e-05, 1.072091680000000e+02]) mcmc_sampler = StandardMetropolis(posterior, scale=1.0, cov=proposal_cov) start = asarray([-11.35, -13.1]) mcmc_params = MCMCParams(start=start, num_iterations=2000) chain = MCMCChain(mcmc_sampler, mcmc_params) chain.append_mcmc_output(StatisticsOutput(print_from=1, lag=1)) home = expanduser("~") folder = os.sep.join([home, modulename]) store_chain_output = StoreChainOutput(folder) chain.append_mcmc_output(store_chain_output) loaded = store_chain_output.load_last_stored_chain() if loaded is None: logging.info("Running chain from scratch") else: logging.info("Running chain from iteration %d" % loaded.iteration) chain = loaded chain.run() f = open(folder + os.sep + "final_chain", "w") dump(chain, f) f.close()
def emp_quantiles(self, X, quantiles=arange(0.1, 1, 0.1)): # need inverse chi2 cdf with self.dimension degrees of freedom chi2_instance = chi2(self.dimension) cutoffs = chi2_instance.isf(1 - quantiles) # whitening D, U = eig(self.L.dot(self.L.T)) D = D ** (-0.5) W = (diag(D).dot(U.T).dot((X - self.mu).T)).T norms_squared = array([norm(w) ** 2 for w in W]) results = zeros([len(quantiles)]) for jj in range(0, len(quantiles)): results[jj] = mean(norms_squared < cutoffs[jj]) return results
def log_pdf(self, X): assert(len(shape(X)) == 2) assert(shape(X)[1] == self.dimension) log_determinant_part = -sum(log(diag(self.L))) quadratic_parts = zeros(len(X)) for i in range(len(X)): x = X[i] - self.mu # solve y=K^(-1)x = L^(-T)L^(-1)x y = solve_triangular(self.L, x.T, lower=True) y = solve_triangular(self.L.T, y, lower=False) quadratic_parts[i] = -0.5 * x.dot(y) const_part = -0.5 * len(self.L) * log(2 * pi) return const_part + log_determinant_part + quadratic_parts
def main(): Log.set_loglevel(logging.DEBUG) prior = Gaussian(Sigma=eye(2) * 100) num_estimates = 2 home = expanduser("~") folder = os.sep.join([home, "sample_ozone_posterior_rr_sge"]) computation_engine = SerialComputationEngine() rr_instance = RussianRoulette(1e-3, block_size=10) posterior = OzonePosteriorRREngine( rr_instance=rr_instance, computation_engine=computation_engine, num_estimates=num_estimates, prior=prior ) posterior.logdet_method = "shogun_estimate" proposal_cov = diag([4.000000000000000e-05, 1.072091680000000e02]) mcmc_sampler = StandardMetropolis(posterior, scale=1.0, cov=proposal_cov) start = asarray([-11.35, -13.1]) mcmc_params = MCMCParams(start=start, num_iterations=200) chain = MCMCChain(mcmc_sampler, mcmc_params) # chain.append_mcmc_output(PlottingOutput(None, plot_from=1, lag=1)) chain.append_mcmc_output(StatisticsOutput(print_from=1, lag=1)) store_chain_output = StoreChainOutput(folder, lag=50) chain.append_mcmc_output(store_chain_output) loaded = store_chain_output.load_last_stored_chain() if loaded is None: logging.info("Running chain from scratch") else: logging.info("Running chain from iteration %d" % loaded.iteration) chain = loaded chain.run() f = open(folder + os.sep + "final_chain", "w") dump(chain, f) f.close()
def roots(p): """ Return the roots of the polynomial coefficients in p. The values in the rank-1 array p are coefficients of a polynomial. If the length of p is n+1 then the polynomial is p[0] * x**n + p[1] * x**(n-1) + ... + p[n-1]*x + p[n] """ # If input is scalar, this makes it an array p = atleast_1d(p) if len(p.shape) != 1: raise ValueError,"Input must be a rank-1 array." # find non-zero array entries non_zero = NX.nonzero(NX.ravel(p))[0] # Return an empty array if polynomial is all zeros if len(non_zero) == 0: return NX.array([]) # find the number of trailing zeros -- this is the number of roots at 0. trailing_zeros = len(p) - non_zero[-1] - 1 # strip leading and trailing zeros p = p[int(non_zero[0]):int(non_zero[-1])+1] # casting: if incoming array isn't floating point, make it floating point. if not issubclass(p.dtype.type, (NX.floating, NX.complexfloating)): p = p.astype(float) N = len(p) if N > 1: # build companion matrix and find its eigenvalues (the roots) A = diag(NX.ones((N-2,), p.dtype), -1) A[0, :] = -p[1:] / p[0] roots = _eigvals(A) else: roots = NX.array([]) # tack any zeros onto the back of the array roots = hstack((roots, NX.zeros(trailing_zeros, roots.dtype))) return roots
def roots(p): """ Return the roots of a polynomial with coefficients given in p. The values in the rank-1 array `p` are coefficients of a polynomial. If the length of `p` is n+1 then the polynomial is described by:: p[0] * x**n + p[1] * x**(n-1) + ... + p[n-1]*x + p[n] Parameters ---------- p : array_like Rank-1 array of polynomial coefficients. Returns ------- out : ndarray An array containing the complex roots of the polynomial. Raises ------ ValueError : When `p` cannot be converted to a rank-1 array. See also -------- poly : Find the coefficients of a polynomial with a given sequence of roots. polyval : Evaluate a polynomial at a point. polyfit : Least squares polynomial fit. poly1d : A one-dimensional polynomial class. Notes ----- The algorithm relies on computing the eigenvalues of the companion matrix [1]_. References ---------- .. [1] R. A. Horn & C. R. Johnson, *Matrix Analysis*. Cambridge, UK: Cambridge University Press, 1999, pp. 146-7. Examples -------- >>> coeff = [3.2, 2, 1] >>> np.roots(coeff) array([-0.3125+0.46351241j, -0.3125-0.46351241j]) """ # If input is scalar, this makes it an array p = atleast_1d(p) if len(p.shape) != 1: raise ValueError("Input must be a rank-1 array.") # find non-zero array entries non_zero = NX.nonzero(NX.ravel(p))[0] # Return an empty array if polynomial is all zeros if len(non_zero) == 0: return NX.array([]) # find the number of trailing zeros -- this is the number of roots at 0. trailing_zeros = len(p) - non_zero[-1] - 1 # strip leading and trailing zeros p = p[int(non_zero[0]):int(non_zero[-1])+1] # casting: if incoming array isn't floating point, make it floating point. if not issubclass(p.dtype.type, (NX.floating, NX.complexfloating)): p = p.astype(float) N = len(p) if N > 1: # build companion matrix and find its eigenvalues (the roots) A = diag(NX.ones((N-2,), p.dtype), -1) A[0, :] = -p[1:] / p[0] roots = eigvals(A) else: roots = NX.array([]) # tack any zeros onto the back of the array roots = hstack((roots, NX.zeros(trailing_zeros, roots.dtype))) return roots
def roots(p): """ Return the roots of a polynomial with coefficients given in p. The values in the rank-1 array `p` are coefficients of a polynomial. If the length of `p` is n+1 then the polynomial is described by p[0] * x**n + p[1] * x**(n-1) + ... + p[n-1]*x + p[n] Parameters ---------- p : array_like of shape(M,) Rank-1 array of polynomial co-efficients. Returns ------- out : ndarray An array containing the complex roots of the polynomial. Raises ------ ValueError: When `p` cannot be converted to a rank-1 array. Examples -------- >>> coeff = [3.2, 2, 1] >>> print np.roots(coeff) [-0.3125+0.46351241j -0.3125-0.46351241j] """ # If input is scalar, this makes it an array p = atleast_1d(p) if len(p.shape) != 1: raise ValueError, "Input must be a rank-1 array." # find non-zero array entries non_zero = NX.nonzero(NX.ravel(p))[0] # Return an empty array if polynomial is all zeros if len(non_zero) == 0: return NX.array([]) # find the number of trailing zeros -- this is the number of roots at 0. trailing_zeros = len(p) - non_zero[-1] - 1 # strip leading and trailing zeros p = p[int(non_zero[0]):int(non_zero[-1]) + 1] # casting: if incoming array isn't floating point, make it floating point. if not issubclass(p.dtype.type, (NX.floating, NX.complexfloating)): p = p.astype(float) N = len(p) if N > 1: # build companion matrix and find its eigenvalues (the roots) A = diag(NX.ones((N - 2, ), p.dtype), -1) A[0, :] = -p[1:] / p[0] roots = eigvals(A) else: roots = NX.array([]) # tack any zeros onto the back of the array roots = hstack((roots, NX.zeros(trailing_zeros, roots.dtype))) return roots
def GPUCB(func = f, kernel = DoubleExponential, params_dist = {'x': Uniform(start = 0, end = 5)}, prev_X = None, prev_y = None, sig = .1, mu_prior = 0, sigma_prior = 1, n_grid = 100, n_iter = 10, seed = 2, time_budget = 36000): time_start = time.time() np.random.seed(seed) n_params = len(params_dist) params_name = params_dist.keys() grid, grid_scaled = GetRandGrid(n_grid, params_dist) mu = np.zeros(n_grid) + mu_prior sigma = np.ones(n_grid)*sigma_prior X = np.empty((n_iter, n_params)) X_scaled = np.matrix(np.empty((n_iter, n_params))) y = np.empty(n_iter) logger.info("%4s |%9s |%9s |%s", "Iter", "Func", "Max", '|'.join(['{:6s}'.format(i) for i in params_name])) for i in xrange(n_iter): #beta = 2*np.log((i+1)**2*2*np.pi**2/3/.1) + \ # 2*n_params*np.log((i+1)**2*n_params) beta = (i+1)**2 #ipdb.set_trace() idx = np.argmax(mu + np.sqrt(beta)*sigma) X[i,:] = grid[idx] X_scaled[i] = grid_scaled[idx] y[i] = func(**dict(zip(params_name, X[i]))) invKT = inv(kernel(X_scaled[:i+1], X_scaled[:i+1])*sigma_prior**2 + sig**2*identity(i + 1)) grid, grid_scaled = GetRandGrid(n_grid, params_dist) kT = kernel(X_scaled[:i+1], grid_scaled)*sigma_prior**2 mu = mu_prior + kT.T.dot(invKT).dot(y[:i+1] - mu_prior) sigma2 = np.ones(n_grid)*sigma_prior**2 - diag(kT.T.dot(invKT).dot(kT)) sigma = np.sqrt(sigma2) logger.info("%4d |%9.4g |%9.4g |%s" , i, y[i], np.max(y[:i+1]), '|'.join(['{:6.2g}'.format(i) for i in X[i]])) if time.time() - time_start > time_budget: break ipdb.set_trace() if True: figure(1); plt.clf(); xlim((0,5)); ylim(-4,10); index = np.argsort(grid[:,0]) gr = grid[:,0] plot(gr[index], mu[index], color = 'red', label = "Mean") plot(gr[index], mu[index] + sigma[index], color = 'blue', label = "Mean + Sigma") plot(gr[index], mu[index] - sigma[index], color = 'blue', label = "Mean - Sigma") plot(X[:i+1,0], y[:i+1], 'o', color = 'green', label = "Eval Points") plot(np.linspace(0,5, num = 500),func(np.linspace(0,5, num = 500)), color = 'green', label = "True Func") plot(gr[index], mu[index] + sqrt(beta)*sigma[index], color = 'yellow', label = "Mean + sqrt(B)*Sigma") plt.grid() legend(loc = 2) show() return {'X': X, 'y': y, 'mu': mu, 'beta': beta, 'sigma': sigma, 'grid': grid}
def roots(p): """ Return the roots of a polynomial with coefficients given in p. The values in the rank-1 array `p` are coefficients of a polynomial. If the length of `p` is n+1 then the polynomial is described by p[0] * x**n + p[1] * x**(n-1) + ... + p[n-1]*x + p[n] Parameters ---------- p : array_like of shape(M,) Rank-1 array of polynomial co-efficients. Returns ------- out : ndarray An array containing the complex roots of the polynomial. Raises ------ ValueError: When `p` cannot be converted to a rank-1 array. Examples -------- >>> coeff = [3.2, 2, 1] >>> print np.roots(coeff) [-0.3125+0.46351241j -0.3125-0.46351241j] """ # If input is scalar, this makes it an array p = atleast_1d(p) if len(p.shape) != 1: raise ValueError,"Input must be a rank-1 array." # find non-zero array entries non_zero = NX.nonzero(NX.ravel(p))[0] # Return an empty array if polynomial is all zeros if len(non_zero) == 0: return NX.array([]) # find the number of trailing zeros -- this is the number of roots at 0. trailing_zeros = len(p) - non_zero[-1] - 1 # strip leading and trailing zeros p = p[int(non_zero[0]):int(non_zero[-1])+1] # casting: if incoming array isn't floating point, make it floating point. if not issubclass(p.dtype.type, (NX.floating, NX.complexfloating)): p = p.astype(float) N = len(p) if N > 1: # build companion matrix and find its eigenvalues (the roots) A = diag(NX.ones((N-2,), p.dtype), -1) A[0, :] = -p[1:] / p[0] roots = eigvals(A) else: roots = NX.array([]) # tack any zeros onto the back of the array roots = hstack((roots, NX.zeros(trailing_zeros, roots.dtype))) return roots
def roots(p): """ Return the roots of a polynomial with coefficients given in p. The values in the rank-1 array `p` are coefficients of a polynomial. If the length of `p` is n+1 then the polynomial is described by:: p[0] * x**n + p[1] * x**(n-1) + ... + p[n-1]*x + p[n] Parameters ---------- p : array_like Rank-1 array of polynomial coefficients. Returns ------- out : ndarray An array containing the complex roots of the polynomial. Raises ------ ValueError : When `p` cannot be converted to a rank-1 array. See also -------- poly : Find the coefficients of a polynomial with a given sequence of roots. polyval : Evaluate a polynomial at a point. polyfit : Least squares polynomial fit. poly1d : A one-dimensional polynomial class. Notes ----- The algorithm relies on computing the eigenvalues of the companion matrix [1]_. References ---------- .. [1] R. A. Horn & C. R. Johnson, *Matrix Analysis*. Cambridge, UK: Cambridge University Press, 1999, pp. 146-7. Examples -------- >>> coeff = [3.2, 2, 1] >>> np.roots(coeff) array([-0.3125+0.46351241j, -0.3125-0.46351241j]) """ # If input is scalar, this makes it an array p = atleast_1d(p) if len(p.shape) != 1: raise ValueError, "Input must be a rank-1 array." # find non-zero array entries non_zero = NX.nonzero(NX.ravel(p))[0] # Return an empty array if polynomial is all zeros if len(non_zero) == 0: return NX.array([]) # find the number of trailing zeros -- this is the number of roots at 0. trailing_zeros = len(p) - non_zero[-1] - 1 # strip leading and trailing zeros p = p[int(non_zero[0]):int(non_zero[-1]) + 1] # casting: if incoming array isn't floating point, make it floating point. if not issubclass(p.dtype.type, (NX.floating, NX.complexfloating)): p = p.astype(float) N = len(p) if N > 1: # build companion matrix and find its eigenvalues (the roots) A = diag(NX.ones((N - 2, ), p.dtype), -1) A[0, :] = -p[1:] / p[0] roots = eigvals(A) else: roots = NX.array([]) # tack any zeros onto the back of the array roots = hstack((roots, NX.zeros(trailing_zeros, roots.dtype))) return roots