def __init__(self, mu=array([0, 0]), Sigma=eye(2), is_cholesky=False, ell=None): Distribution.__init__(self, len(Sigma)) assert (len(shape(mu)) == 1) assert (max(shape(Sigma)) == len(mu)) self.mu = mu self.ell = ell if is_cholesky: self.L = Sigma if ell == None: assert (shape(Sigma)[0] == shape(Sigma)[1]) else: assert (shape(Sigma)[1] == ell) else: assert (shape(Sigma)[0] == shape(Sigma)[1]) if ell is not None: self.L, _, _ = MatrixTools.low_rank_approx(Sigma, ell) self.L = self.L.T assert (shape(self.L)[1] == ell) else: try: self.L = cholesky(Sigma) except LinAlgError: # some really crude check for PSD (which only corrects for orunding errors self.L = cholesky(Sigma + eye(len(Sigma)) * 1e-5)
def log_pdf(self, X): assert (len(shape(X)) == 2) assert (shape(X)[1] == self.dimension) # compute all norms norms = array([norm(x) for x in X]) # compute angles (second component first first) angles = arctan2(X[:, 1], X[:, 0]) # gaussian parameters mu = self.radius + self.amplitude * cos(self.frequency * angles) log_pdf2d = zeros(len(X)) gaussian = Gaussian(array([mu[0]]), array([[self.variance]])) for i in range(len(X)): gaussian.mu = mu[i] log_pdf2d[i] = gaussian.log_pdf(array([[norms[i]]])) if self.dimension > 2: remain_dims = Gaussian(zeros([self.dimension - 2]), eye(self.dimension - 2)) log_pdfoverall = log_pdf2d + remain_dims.log_pdf( X[:, 2:self.dimension]) else: log_pdfoverall = log_pdf2d return log_pdfoverall
def find_mode_newton(self, return_full=False): """ Newton search for mode of p(y|f)p(f) from GP book, algorithm 3.1, added step size """ K = self.gp.K if self.newton_start is None: f = zeros(len(K)) else: f = self.newton_start if return_full: steps = [f] iteration = 0 norm_difference = inf objective_value = -inf while iteration < self.newton_max_iterations and norm_difference > self.newton_epsilon: # from GP book, algorithm 3.1, added step size # scale log_lik_grad_vector and K^-1 f = a w = -self.gp.likelihood.log_lik_hessian_vector(self.gp.y, f) w_sqrt = sqrt(w) # diag(w_sqrt).dot(K.dot(diag(w_sqrt))) == (K.T*w_sqrt).T*w_sqrt L = cholesky(eye(len(K)) + (K.T * w_sqrt).T * w_sqrt) b = f * w + self.newton_step * self.gp.likelihood.log_lik_grad_vector(self.gp.y, f) # a=b-diag(w_sqrt).dot(inv(eye(len(K)) + (K.T*w_sqrt).T*w_sqrt).dot(diag(w_sqrt).dot(K.dot(b)))) a = w_sqrt * (K.dot(b)) a = solve_triangular(L, a, lower=True) a = solve_triangular(L.T, a, lower=False) a = w_sqrt * a a = b - a f_new = K.dot(self.newton_step * a) # convergence stuff and next iteration objective_value_new = -0.5 * a.T.dot(f) + sum(self.gp.likelihood.log_lik_vector(self.gp.y, f)) norm_difference = norm(f - f_new) if objective_value_new > objective_value: f = f_new if return_full: steps.append(f) else: self.newton_step /= 2 iteration += 1 objective_value = objective_value_new self.computed = True if return_full: return f, L, asarray(steps) else: return f
def main(): Log.set_loglevel(logging.DEBUG) prior = Gaussian(Sigma=eye(2) * 100) posterior = OzonePosterior(prior, logdet_alg="scikits", solve_method="scikits") proposal_cov = diag([ 4.000000000000000e-05, 1.072091680000000e+02]) mcmc_sampler = StandardMetropolis(posterior, scale=1.0, cov=proposal_cov) start = asarray([-11.35, -13.1]) mcmc_params = MCMCParams(start=start, num_iterations=5000) chain = MCMCChain(mcmc_sampler, mcmc_params) chain.append_mcmc_output(StatisticsOutput(print_from=1, lag=1)) home = expanduser("~") folder = os.sep.join([home, "sample_ozone_posterior_average_serial"]) store_chain_output = StoreChainOutput(folder) chain.append_mcmc_output(store_chain_output) loaded = store_chain_output.load_last_stored_chain() if loaded is None: logging.info("Running chain from scratch") else: logging.info("Running chain from iteration %d" % loaded.iteration) chain = loaded chain.run() f = open(folder + os.sep + "final_chain", "w") dump(chain, f) f.close()
def main(): Log.set_loglevel(logging.DEBUG) prior = Gaussian(Sigma=eye(2) * 100) posterior = OzonePosterior(prior, logdet_alg="scikits", solve_method="scikits") proposal_cov = diag([4.000000000000000e-05, 1.072091680000000e+02]) mcmc_sampler = StandardMetropolis(posterior, scale=1.0, cov=proposal_cov) start = asarray([-11.35, -13.1]) mcmc_params = MCMCParams(start=start, num_iterations=5000) chain = MCMCChain(mcmc_sampler, mcmc_params) chain.append_mcmc_output(StatisticsOutput(print_from=1, lag=1)) home = expanduser("~") folder = os.sep.join([home, "sample_ozone_posterior_average_serial"]) store_chain_output = StoreChainOutput(folder) chain.append_mcmc_output(store_chain_output) loaded = store_chain_output.load_last_stored_chain() if loaded is None: logging.info("Running chain from scratch") else: logging.info("Running chain from iteration %d" % loaded.iteration) chain = loaded chain.run() f = open(folder + os.sep + "final_chain", "w") dump(chain, f) f.close()
def log_pdf(self, X): assert(len(shape(X)) == 2) assert(shape(X)[1] == self.dimension) transformed = X.copy() transformed[:, 1] = X[:, 1] - self.bananicity * ((X[:, 0] ** 2) - self.V) transformed[:, 0] = X[:, 0] / sqrt(self.V) phi = Gaussian(zeros([self.dimension]), eye(self.dimension)) return phi.log_pdf(transformed)
def emp_quantiles(self, X, quantiles=arange(0.1, 1, 0.1)): norms = array([norm(x) for x in X]) angles = arctan2(X[:, 1], X[:, 0]) mu = self.radius + self.amplitude * cos(self.frequency * angles) transformed = hstack((array([norms - mu]).T, X[:, 2:self.dimension])) cov = eye(self.dimension - 1) cov[0, 0] = self.variance gaussian = Gaussian(zeros([self.dimension - 1]), cov) return gaussian.emp_quantiles(transformed)
def log_pdf(self, X): assert (len(shape(X)) == 2) assert (shape(X)[1] == self.dimension) transformed = X.copy() transformed[:, 1] = X[:, 1] - self.bananicity * ((X[:, 0]**2) - self.V) transformed[:, 0] = X[:, 0] / sqrt(self.V) phi = Gaussian(zeros([self.dimension]), eye(self.dimension)) return phi.log_pdf(transformed)
def emp_quantiles(self, X, quantiles=arange(0.1, 1, 0.1)): norms = array([norm(x) for x in X]) angles = arctan2(X[:, 1], X[:, 0]) mu = self.radius + self.amplitude * cos(self.frequency * angles) transformed = hstack((array([norms-mu]).T, X[:,2:self.dimension])) cov=eye(self.dimension-1) cov[0,0]=self.variance gaussian=Gaussian(zeros([self.dimension-1]), cov) return gaussian.emp_quantiles(transformed)
def main(): Log.set_loglevel(logging.DEBUG) prior = Gaussian(Sigma=eye(2) * 100) num_estimates = 1000 home = expanduser("~") folder = os.sep.join([home, "sample_ozone_posterior_rr_sge"]) # cluster admin set project jump for me to exclusively allocate nodes parameter_prefix = "" # #$ -P jump" cluster_parameters = BatchClusterParameters( foldername=folder, memory=7.8, loglevel=logging.DEBUG, parameter_prefix=parameter_prefix, max_walltime=60 * 60 * 24 - 1) computation_engine = SGEComputationEngine(cluster_parameters, check_interval=10) rr_instance = RussianRoulette(1e-3, block_size=400) posterior = OzonePosteriorRREngine(rr_instance=rr_instance, computation_engine=computation_engine, num_estimates=num_estimates, prior=prior) posterior.logdet_method = "shogun_estimate" proposal_cov = diag([4.000000000000000e-05, 1.072091680000000e+02]) mcmc_sampler = StandardMetropolis(posterior, scale=1.0, cov=proposal_cov) start = asarray([-11.55, -10.1]) mcmc_params = MCMCParams(start=start, num_iterations=5000) chain = MCMCChain(mcmc_sampler, mcmc_params) # chain.append_mcmc_output(PlottingOutput(None, plot_from=1, lag=1)) chain.append_mcmc_output(StatisticsOutput(print_from=1, lag=1)) store_chain_output = StoreChainOutput(folder, lag=1) chain.append_mcmc_output(store_chain_output) loaded = store_chain_output.load_last_stored_chain() if loaded is None: logging.info("Running chain from scratch") else: logging.info("Running chain from iteration %d" % loaded.iteration) chain = loaded chain.run() f = open(folder + os.sep + "final_chain", "w") dump(chain, f) f.close()
def __init__(self, distribution, scale=None, cov=None): MCMCSampler.__init__(self, distribution) if scale is None: self.scale = (2.38 ** 2) / distribution.dimension else: self.scale = scale if cov is None: self.cov = eye(distribution.dimension) else: self.cov = cov
def __init__(self, distribution, scale=None, cov=None): MCMCSampler.__init__(self, distribution) if scale is None: self.scale = (2.38**2) / distribution.dimension else: self.scale = scale if cov is None: self.cov = eye(distribution.dimension) else: self.cov = cov
def main(): Log.set_loglevel(logging.DEBUG) modulename = "sample_ozone_posterior_average_slurm" if not FileSystem.cmd_exists("sbatch"): engine = SerialComputationEngine() else: johns_slurm_hack = "#SBATCH --partition=intel-ivy,wrkstn,compute" johns_slurm_hack = "#SBATCH --partition=intel-ivy,compute" folder = os.sep + os.sep.join(["nfs", "data3", "ucabhst", modulename]) batch_parameters = BatchClusterParameters( foldername=folder, max_walltime=24 * 60 * 60, resubmit_on_timeout=False, memory=3, parameter_prefix=johns_slurm_hack) engine = SlurmComputationEngine(batch_parameters, check_interval=1, do_clean_up=True) prior = Gaussian(Sigma=eye(2) * 100) num_estimates = 100 posterior = OzonePosteriorAverageEngine(computation_engine=engine, num_estimates=num_estimates, prior=prior) posterior.logdet_method = "shogun_estimate" proposal_cov = diag([4.000000000000000e-05, 1.072091680000000e+02]) mcmc_sampler = StandardMetropolis(posterior, scale=1.0, cov=proposal_cov) start = asarray([-11.35, -13.1]) mcmc_params = MCMCParams(start=start, num_iterations=2000) chain = MCMCChain(mcmc_sampler, mcmc_params) chain.append_mcmc_output(StatisticsOutput(print_from=1, lag=1)) home = expanduser("~") folder = os.sep.join([home, modulename]) store_chain_output = StoreChainOutput(folder) chain.append_mcmc_output(store_chain_output) loaded = store_chain_output.load_last_stored_chain() if loaded is None: logging.info("Running chain from scratch") else: logging.info("Running chain from iteration %d" % loaded.iteration) chain = loaded chain.run() f = open(folder + os.sep + "final_chain", "w") dump(chain, f) f.close()
def __init__(self, mu=asarray([0, 0]), Sigma=eye(2), is_cholesky=False): DensityFunction.__init__(self, len(Sigma)) assert(len(shape(mu)) == 1) assert(max(shape(Sigma)) == len(mu)) self.mu = mu if is_cholesky: self.L = Sigma else: assert(shape(Sigma)[0] == shape(Sigma)[1]) self.L = cholesky(Sigma)
def main(): Log.set_loglevel(logging.DEBUG) prior = Gaussian(Sigma=eye(2) * 100) num_estimates = 1000 home = expanduser("~") folder = os.sep.join([home, "sample_ozone_posterior_rr_sge"]) # cluster admin set project jump for me to exclusively allocate nodes parameter_prefix = "" # #$ -P jump" cluster_parameters = BatchClusterParameters(foldername=folder, memory=7.8, loglevel=logging.DEBUG, parameter_prefix=parameter_prefix, max_walltime=60 * 60 * 24 - 1) computation_engine = SGEComputationEngine(cluster_parameters, check_interval=10) rr_instance = RussianRoulette(1e-3, block_size=400) posterior = OzonePosteriorRREngine(rr_instance=rr_instance, computation_engine=computation_engine, num_estimates=num_estimates, prior=prior) posterior.logdet_method = "shogun_estimate" proposal_cov = diag([ 4.000000000000000e-05, 1.072091680000000e+02]) mcmc_sampler = StandardMetropolis(posterior, scale=1.0, cov=proposal_cov) start = asarray([-11.55, -10.1]) mcmc_params = MCMCParams(start=start, num_iterations=5000) chain = MCMCChain(mcmc_sampler, mcmc_params) # chain.append_mcmc_output(PlottingOutput(None, plot_from=1, lag=1)) chain.append_mcmc_output(StatisticsOutput(print_from=1, lag=1)) store_chain_output = StoreChainOutput(folder, lag=1) chain.append_mcmc_output(store_chain_output) loaded = store_chain_output.load_last_stored_chain() if loaded is None: logging.info("Running chain from scratch") else: logging.info("Running chain from iteration %d" % loaded.iteration) chain = loaded chain.run() f = open(folder + os.sep + "final_chain", "w") dump(chain, f) f.close()
def __init__(self, distribution, num_eigen=2, \ mean_est=array([-2.0, -2.0]), cov_est=0.05 * eye(2), \ sample_discard=500, sample_lag=10, accstar=0.234): AdaptiveMetropolis.__init__(self, distribution=distribution, \ mean_est=mean_est, cov_est=cov_est, \ sample_discard=sample_discard, sample_lag=sample_lag, accstar=accstar) assert (num_eigen <= distribution.dimension) self.num_eigen = num_eigen self.dwscale = self.globalscale * ones([self.num_eigen]) u, s, _ = svd(self.cov_est) self.eigvalues = s[0:self.num_eigen] self.eigvectors = u[:, 0:self.num_eigen]
def main(): Log.set_loglevel(logging.DEBUG) modulename = "sample_ozone_posterior_average_slurm" if not FileSystem.cmd_exists("sbatch"): engine = SerialComputationEngine() else: johns_slurm_hack = "#SBATCH --partition=intel-ivy,wrkstn,compute" johns_slurm_hack = "#SBATCH --partition=intel-ivy,compute" folder = os.sep + os.sep.join(["nfs", "data3", "ucabhst", modulename]) batch_parameters = BatchClusterParameters(foldername=folder, max_walltime=24 * 60 * 60, resubmit_on_timeout=False, memory=3, parameter_prefix=johns_slurm_hack) engine = SlurmComputationEngine(batch_parameters, check_interval=1, do_clean_up=True) prior = Gaussian(Sigma=eye(2) * 100) num_estimates = 100 posterior = OzonePosteriorAverageEngine(computation_engine=engine, num_estimates=num_estimates, prior=prior) posterior.logdet_method = "shogun_estimate" proposal_cov = diag([ 4.000000000000000e-05, 1.072091680000000e+02]) mcmc_sampler = StandardMetropolis(posterior, scale=1.0, cov=proposal_cov) start = asarray([-11.35, -13.1]) mcmc_params = MCMCParams(start=start, num_iterations=2000) chain = MCMCChain(mcmc_sampler, mcmc_params) chain.append_mcmc_output(StatisticsOutput(print_from=1, lag=1)) home = expanduser("~") folder = os.sep.join([home, modulename]) store_chain_output = StoreChainOutput(folder) chain.append_mcmc_output(store_chain_output) loaded = store_chain_output.load_last_stored_chain() if loaded is None: logging.info("Running chain from scratch") else: logging.info("Running chain from iteration %d" % loaded.iteration) chain = loaded chain.run() f = open(folder + os.sep + "final_chain", "w") dump(chain, f) f.close()
def __init__(self, dimension=2, num_components=2, components=None, mixing_proportion=None): Distribution.__init__(self, dimension) self.num_components = num_components if (components == None): self.components = [Gaussian(mu=zeros(self.dimension),Sigma=eye(self.dimension)) for _ in range(self.num_components)] else: assert(len(components)==self.num_components) self.components=components if (mixing_proportion == None): self.mixing_proportion=Discrete((1.0/num_components)*ones([num_components])) else: assert(num_components==mixing_proportion.num_objects) self.mixing_proportion = mixing_proportion
def emp_quantiles(self, X, quantiles=arange(0.1, 1, 0.1)): assert(len(shape(X)) == 2) assert(shape(X)[1] == self.dimension) substract=self.bananicity * ((X[:, 0] ** 2) - self.V) divide=sqrt(self.V) X[:, 1] -= substract X[:, 0] /= divide phi = Gaussian(zeros([self.dimension]), eye(self.dimension)) quantiles=phi.emp_quantiles(X, quantiles) # undo changes to X X[:, 0] *= divide X[:, 1] += substract return quantiles
def emp_quantiles(self, X, quantiles=arange(0.1, 1, 0.1)): assert (len(shape(X)) == 2) assert (shape(X)[1] == self.dimension) substract = self.bananicity * ((X[:, 0]**2) - self.V) divide = sqrt(self.V) X[:, 1] -= substract X[:, 0] /= divide phi = Gaussian(zeros([self.dimension]), eye(self.dimension)) quantiles = phi.emp_quantiles(X, quantiles) # undo changes to X X[:, 0] *= divide X[:, 1] += substract return quantiles
def main(): Log.set_loglevel(logging.DEBUG) prior = Gaussian(Sigma=eye(2) * 100) num_estimates = 2 home = expanduser("~") folder = os.sep.join([home, "sample_ozone_posterior_rr_sge"]) computation_engine = SerialComputationEngine() rr_instance = RussianRoulette(1e-3, block_size=10) posterior = OzonePosteriorRREngine(rr_instance=rr_instance, computation_engine=computation_engine, num_estimates=num_estimates, prior=prior) posterior.logdet_method = "shogun_estimate" proposal_cov = diag([4.000000000000000e-05, 1.072091680000000e+02]) mcmc_sampler = StandardMetropolis(posterior, scale=1.0, cov=proposal_cov) start = asarray([-11.35, -13.1]) mcmc_params = MCMCParams(start=start, num_iterations=200) chain = MCMCChain(mcmc_sampler, mcmc_params) # chain.append_mcmc_output(PlottingOutput(None, plot_from=1, lag=1)) chain.append_mcmc_output(StatisticsOutput(print_from=1, lag=1)) store_chain_output = StoreChainOutput(folder, lag=50) chain.append_mcmc_output(store_chain_output) loaded = store_chain_output.load_last_stored_chain() if loaded is None: logging.info("Running chain from scratch") else: logging.info("Running chain from iteration %d" % loaded.iteration) chain = loaded chain.run() f = open(folder + os.sep + "final_chain", "w") dump(chain, f) f.close()
def main(): Log.set_loglevel(logging.DEBUG) prior = Gaussian(Sigma=eye(2) * 100) num_estimates = 2 home = expanduser("~") folder = os.sep.join([home, "sample_ozone_posterior_rr_sge"]) computation_engine = SerialComputationEngine() rr_instance = RussianRoulette(1e-3, block_size=10) posterior = OzonePosteriorRREngine( rr_instance=rr_instance, computation_engine=computation_engine, num_estimates=num_estimates, prior=prior ) posterior.logdet_method = "shogun_estimate" proposal_cov = diag([4.000000000000000e-05, 1.072091680000000e02]) mcmc_sampler = StandardMetropolis(posterior, scale=1.0, cov=proposal_cov) start = asarray([-11.35, -13.1]) mcmc_params = MCMCParams(start=start, num_iterations=200) chain = MCMCChain(mcmc_sampler, mcmc_params) # chain.append_mcmc_output(PlottingOutput(None, plot_from=1, lag=1)) chain.append_mcmc_output(StatisticsOutput(print_from=1, lag=1)) store_chain_output = StoreChainOutput(folder, lag=50) chain.append_mcmc_output(store_chain_output) loaded = store_chain_output.load_last_stored_chain() if loaded is None: logging.info("Running chain from scratch") else: logging.info("Running chain from iteration %d" % loaded.iteration) chain = loaded chain.run() f = open(folder + os.sep + "final_chain", "w") dump(chain, f) f.close()
def construct_proposal(self, y): """ proposal is a mixture of normals, centred at y and with covariance gamma^2 I + nu^2 MHaa'HM', where a are the eigenvectors of centred kernel matrix Kc=HKH """ assert (len(shape(y)) == 1) m = MixtureDistribution(self.distribution.dimension, self.num_eigen) m.mixing_proportion = Discrete( (self.eigvalues + 1) / (sum(self.eigvalues) + self.num_eigen)) # print "current mixing proportion: ", m.mixing_proportion.omega M = 2 * self.kernel.gradient(y, self.Z) H = Kernel.centring_matrix(len(self.Z)) for ii in range(self.num_eigen): Sigma = self.gamma ** 2 * eye(len(y)) + \ self.nu2 * (M.T).dot(H.dot(outer(self.eigvectors[:, ii], self.eigvectors[:, ii]).dot(H.dot(M)))) m.components[ii] = Gaussian(y, Sigma) return m
def construct_proposal(self, y): """ proposal is a mixture of normals, centred at y and with covariance gamma^2 I + nu^2 MHaa'HM', where a are the eigenvectors of centred kernel matrix Kc=HKH """ assert len(shape(y)) == 1 m = MixtureDistribution(self.distribution.dimension, self.num_eigen) m.mixing_proportion = Discrete((self.eigvalues + 1) / (sum(self.eigvalues) + self.num_eigen)) # print "current mixing proportion: ", m.mixing_proportion.omega M = 2 * self.kernel.gradient(y, self.Z) H = Kernel.centring_matrix(len(self.Z)) for ii in range(self.num_eigen): Sigma = self.gamma ** 2 * eye(len(y)) + self.nu2 * (M.T).dot( H.dot(outer(self.eigvectors[:, ii], self.eigvectors[:, ii]).dot(H.dot(M))) ) m.components[ii] = Gaussian(y, Sigma) return m
def TheLeastSquareMethod(X, Y): """ 最小二乘法 """ regula = eye(ORDER + 1) X_matrix = [] for i in range(ORDER + 1): X_matrix.append(array(X)**i) X_matrix = mat(X_matrix).T Y_matrix = array(Y).reshape((len(Y), 1)) X_matrix_T = X_matrix.T #print dot(X_matrix_T,X_matrix) B = dot(dot(dot(X_matrix_T, X_matrix).I, X_matrix_T), Y_matrix) B1 = dot(dot((dot(X_matrix_T, X_matrix) + lamda * regula).I, X_matrix_T), Y_matrix) result = dot(X_matrix, B) result_reg = dot(X_matrix, B1) return X_matrix, Y_matrix, B, result, result_reg, B1
def __init__(self, dimension=2, num_components=2, components=None, mixing_proportion=None): Distribution.__init__(self, dimension) self.num_components = num_components if (components == None): self.components = [ Gaussian(mu=zeros(self.dimension), Sigma=eye(self.dimension)) for _ in range(self.num_components) ] else: assert (len(components) == self.num_components) self.components = components if (mixing_proportion == None): self.mixing_proportion = Discrete( (1.0 / num_components) * ones([num_components])) else: assert (num_components == mixing_proportion.num_objects) self.mixing_proportion = mixing_proportion
def predict(self, X_test, f_mode=None): """ Predictions for GP with Laplace approximation. from GP book, algorithm 3.2, """ if f_mode is None: f_mode = self.find_mode_newton() predictions = zeros(len(X_test)) K = self.gp.K K_train_test = self.gp.covariance.compute(self.gp.X, X_test) w = -self.gp.likelihood.log_lik_hessian_vector(self.gp.y, f_mode) w_sqrt = sqrt(w) # diag(w_sqrt).dot(K.dot(diag(w_sqrt))) == (K.T*w_sqrt).T*w_sqrt L = cholesky(eye(len(K)) + (K.T * w_sqrt).T * w_sqrt) # iterator for all testing points for i in range(len(X_test)): k = K_train_test[:, i] k_self = self.gp.covariance.compute([X_test[i]], [X_test[i]])[0] f_mean = k.dot( self.gp.likelihood.log_lik_grad_vector(self.gp.y, f_mode)) v = solve_triangular(L, w_sqrt * k, lower=True) f_var = k_self - v.T.dot(v) predictions[i] = integrate.quad( lambda x: norm.pdf(x, f_mean, f_var), -inf, inf)[0] # # integrate over Gaussian using some crude numerical integration # samples=randn(1000)*sqrt(f_var) + f_mean # # log_liks=self.gp.likelihood.log_lik_vector(1.0, samples) # predictions[i]=1.0/len(samples)*GPTools.log_sum_exp(log_liks) return predictions
def predict(self, X_test, f_mode=None): """ Predictions for GP with Laplace approximation. from GP book, algorithm 3.2, """ if f_mode is None: f_mode = self.find_mode_newton() predictions = zeros(len(X_test)) K = self.gp.K K_train_test = self.gp.covariance.compute(self.gp.X, X_test) w = -self.gp.likelihood.log_lik_hessian_vector(self.gp.y, f_mode) w_sqrt = sqrt(w) # diag(w_sqrt).dot(K.dot(diag(w_sqrt))) == (K.T*w_sqrt).T*w_sqrt L = cholesky(eye(len(K)) + (K.T * w_sqrt).T * w_sqrt) # iterator for all testing points for i in range(len(X_test)): k = K_train_test[:, i] k_self = self.gp.covariance.compute([X_test[i]], [X_test[i]])[0] f_mean = k.dot(self.gp.likelihood.log_lik_grad_vector(self.gp.y, f_mode)) v = solve_triangular(L, w_sqrt * k, lower=True) f_var = k_self - v.T.dot(v) predictions[i] = integrate.quad(lambda x: norm.pdf(x, f_mean, f_var), -inf, inf)[0] # # integrate over Gaussian using some crude numerical integration # samples=randn(1000)*sqrt(f_var) + f_mean # # log_liks=self.gp.likelihood.log_lik_vector(1.0, samples) # predictions[i]=1.0/len(samples)*GPTools.log_sum_exp(log_liks) return predictions
def test_simple_array_operations(self): a = array([[1.,2.], [3.,4.]]) numpy.testing.assert_array_equal(a.transpose(), array([[1.,3.], [2.,4.]])) numpy.testing.assert_array_almost_equal(trace(a), 5) inv_a = inv(a) b = array([[-2.,1.], [1.5,-.5]]) self.assertTrue(numpy.allclose(inv_a,b)) i = dot(a,inv_a) numpy.testing.assert_array_almost_equal(i, eye(2), 1) numpy.testing.assert_array_almost_equal(inv_a, b) # system of linear equations a = array([[3,2,-1], [2,-2,4], [-1,0.5,-1]]) b = array([1,-2,0]) c = solve(a,b) d = dot(a,c) numpy.testing.assert_array_almost_equal(b, d, 1) a = array([[.8,.3], [.2,.7]]) eigen_values, eigen_vectors = eig(a) lambda_1 = eigen_values[0] x_1 = eigen_vectors[:,0] lambda_2 = eigen_values[1] x_2 = eigen_vectors[:,1]
def log_pdf(self, X): assert(len(shape(X)) == 2) assert(shape(X)[1] == self.dimension) # compute all norms norms = array([norm(x) for x in X]) # compute angles (second component first first) angles = arctan2(X[:, 1], X[:, 0]) # gaussian parameters mu = self.radius + self.amplitude * cos(self.frequency * angles) log_pdf2d = zeros(len(X)) gaussian = Gaussian(array([mu[0]]), array([[self.variance]])) for i in range(len(X)): gaussian.mu = mu[i] log_pdf2d[i] = gaussian.log_pdf(array([[norms[i]]])) if self.dimension>2: remain_dims=Gaussian(zeros([self.dimension-2]), eye(self.dimension-2)) log_pdfoverall=log_pdf2d+remain_dims.log_pdf(X[:,2:self.dimension]) else: log_pdfoverall=log_pdf2d return log_pdfoverall
def find_mode_newton(self, return_full=False): """ Newton search for mode of p(y|f)p(f) from GP book, algorithm 3.1, added step size """ K = self.gp.K if self.newton_start is None: f = zeros(len(K)) else: f = self.newton_start if return_full: steps = [f] iteration = 0 norm_difference = inf objective_value = -inf while iteration < self.newton_max_iterations and norm_difference > self.newton_epsilon: # from GP book, algorithm 3.1, added step size # scale log_lik_grad_vector and K^-1 f = a w = -self.gp.likelihood.log_lik_hessian_vector(self.gp.y, f) w_sqrt = sqrt(w) # diag(w_sqrt).dot(K.dot(diag(w_sqrt))) == (K.T*w_sqrt).T*w_sqrt L = cholesky(eye(len(K)) + (K.T * w_sqrt).T * w_sqrt) b = f * w + self.newton_step * \ self.gp.likelihood.log_lik_grad_vector(self.gp.y, f) # a=b-diag(w_sqrt).dot(inv(eye(len(K)) + (K.T*w_sqrt).T*w_sqrt).dot(diag(w_sqrt).dot(K.dot(b)))) a = (w_sqrt * (K.dot(b))) a = solve_triangular(L, a, lower=True) a = solve_triangular(L.T, a, lower=False) a = w_sqrt * a a = b - a f_new = K.dot(self.newton_step * a) # convergence stuff and next iteration objective_value_new = -0.5 * a.T.dot(f) + \ sum(self.gp.likelihood.log_lik_vector(self.gp.y, f)) norm_difference = norm(f - f_new) if objective_value_new > objective_value: f = f_new if return_full: steps.append(f) else: self.newton_step /= 2 iteration += 1 objective_value = objective_value_new self.computed = True if return_full: return f, L, asarray(steps) else: return f
# store index sets at source and target of every edge index_sets={} print "precomputing systems for messages from observed nodes" graphlab_lines.lines.append(os.linesep + "# edges with observed targets") for node, observation in observations.items(): for out_message in graph[node]: edge=(out_message, node) graphlab_lines.new_edge_observed_target(node, out_message) data_source=data[edge][0] data_target=data[edge][1] Ks_chol, Is, Rs, Ws=incomplete_cholesky(data_source, kernel, eta) Kt_chol, It, Rt, Wt=incomplete_cholesky(data_target, kernel, eta) Qs,Rs,Ps=qr(Ws.dot(Ws.T)+Ks_chol+eye(shape(Ks_chol)[0])*reg_lambda, pivoting=True) Qt,Rt,Pt=qr(Wt.dot(Wt.T)+Kt_chol+eye(shape(Kt_chol)[0])*reg_lambda, pivoting=True) savetxt(graphlab_lines.add_edge(node, out_message,"Q_s"), Qs) savetxt(graphlab_lines.add_edge(node, out_message,"R_s"), Rs) savetxt(graphlab_lines.add_edge(node, out_message,"P_s"), Ps) savetxt(graphlab_lines.add_edge(node, out_message,"Q_t"), Qt) savetxt(graphlab_lines.add_edge(node, out_message,"R_t"), Rt) savetxt(graphlab_lines.add_edge(node, out_message,"P_t"), Pt) savetxt(graphlab_lines.add_edge(node, out_message,"W"), Ws.dot(Wt.T)) print "precomputing systems for messages from non-observed nodes" graphlab_lines.lines.append(os.linesep + "# edges with non-observed targets") for edge in edges:
#coding:utf-8 # ------------------------------------------------------------ # 简介 : 机器学习基础 numpy # # 更新 : 2015年1月7日@ # ------------------------------------------------------------ from numpy import random from numpy.matrixlib.defmatrix import mat from numpy.lib.twodim_base import eye #4*4随机数组 random.rand(4, 4) #数组转矩阵 randmat = mat(random.rand(4, 4)) print(randmat) #.I专为逆矩阵 print(randmat.I) randmatv = randmat.I #矩阵和逆矩阵相乘,对角为1 其他为0 -eye(4)为单位矩阵 相减后看到有误差 print(randmat * randmatv - eye(4))
experiment_dir = experiment_dir_base + os.sep + str( os.path.abspath(sys.argv[0])).split(os.sep)[-1].split(".")[0] + os.sep print "running experiments", n, "times at base", experiment_dir for i in range(n): distribution = Ring() mcmc_samplers = [] kernel = GaussianKernel(sigma=1) # mcmc_samplers.append(KameleonWindow(distribution, kernel)) mcmc_samplers.append(KameleonWindowLearnScale(distribution, kernel)) mean_est = array([-2.0, -2.0]) cov_est = 0.05 * eye(2) # mcmc_samplers.append(AdaptiveMetropolis(distribution, mean_est=mean_est, cov_est=cov_est)) mcmc_samplers.append( AdaptiveMetropolisLearnScale(distribution, mean_est=mean_est, cov_est=cov_est)) num_eigen = 2 mcmc_samplers.append( AdaptiveMetropolisPCA(distribution, num_eigen=num_eigen, mean_est=mean_est, cov_est=cov_est)) mcmc_samplers.append(StandardMetropolis(distribution))
experiment_dir = experiment_dir_base + str(os.path.abspath( sys.argv[0])).split(os.sep)[-1].split(".")[0] + os.sep print "running experiments", n, "times at base", experiment_dir # load data data, labels = GPData.get_glass_data() # normalise and whiten dataset data -= mean(data, 0) L = cholesky(cov(data.T)) data = solve_triangular(L, data.T, lower=True).T dim = shape(data)[1] # prior on theta and posterior target estimate theta_prior = Gaussian(mu=0 * ones(dim), Sigma=eye(dim) * 5) distribution=PseudoMarginalHyperparameterDistribution(data, labels, \ n_importance=100, prior=theta_prior, \ ridge=1e-3) sigma = 23.0 print "using sigma", sigma kernel = GaussianKernel(sigma=sigma) for i in range(n): mcmc_samplers = [] burnin = 20000 num_iterations = 100000
# throw away some data n=250 seed(1) idx=permutation(len(data)) idx=idx[:n] data=data[idx] labels=labels[idx] # normalise and whiten dataset data-=mean(data, 0) L=cholesky(cov(data.T)) data=solve_triangular(L, data.T, lower=True).T dim=shape(data)[1] # prior on theta and posterior target estimate theta_prior=Gaussian(mu=0*ones(dim), Sigma=eye(dim)*5) target=PseudoMarginalHyperparameterDistribution(data, labels, \ n_importance=100, prior=theta_prior, \ ridge=1e-3) # create sampler burnin=10000 num_iterations=burnin+300000 kernel = GaussianKernel(sigma=23.0) sampler=KameleonWindowLearnScale(target, kernel, stop_adapt=burnin) # sampler=AdaptiveMetropolisLearnScale(target) # sampler=StandardMetropolis(target) # posterior mode derived by initial tests start=zeros(target.dimension) params = MCMCParams(start=start, num_iterations=num_iterations, burnin=burnin)
__all__ = ['matrix_power', 'solve', 'tensorsolve', 'tensorinv', 'inv', 'cholesky', 'eigvals', 'eigvalsh', 'pinv', 'slogdet', 'det', 'svd', 'eig', 'eigh', 'lstsq', 'norm', 'qr', 'cond', 'matrix_rank', 'LinAlgError', 'multi_dot']; import functools import operator import warnings; from numpy.core import ( array, asarray, zeros, empty, empty_like, intc, single, double, csingle, cdouble, inexact, complexfloating, newaxis, all, Inf, dot, add, multiply, sqrt, fastCopyAndTranspose, sum, isfinite, finfo, errstate, geterrobj, moveaxis, amin, amax, product, abs, atleast_2d, intp, asanyarray, object_, matmul, swapaxes, divide, count_nonzero, isnan, sign ); from numpy.core.multiarray import normalize_axis_index; from numpy.core.overrides import set_module; from numpy.core import overrides; from numpy.lib.twodim_base import triu, eye; from numpy.linalg import lapack_lite, _umath_linalg; array_function_dispatch = functools.partial( overrides.array_function_dispatch, module='numpy.linalg'); _N = b'N'; _V = b'V'; _A = b'A'; _S = b'S'; _L = b'L' fortran_int = intc @set_module('numpy.linalg') class LinAlgError(Exception): def _determine_error_states(): errobj = geterrobj() bufsize = errobj[0] with errstate(invalid='call', over='ignore', divide='ignore', under='ignore'): invalid_call_errmask = geterrobj()[1] return [bufsize, invalid_call_errmask, None]; _linalg_error_extobj = _determine_error_states(); del _determine_error_states; def _raise_linalgerror_singular(err, flag): raise LinAlgError("Singular matrix"); def _raise_linalgerror_nonposdef(err, flag): raise LinAlgError("Matrix is not positive definite"); def _raise_linalgerror_eigenvalues_nonconvergence(err, flag): raise LinAlgError("Eigenvalues did not converge"); def _raise_linalgerror_svd_nonconvergence(err, flag): raise LinAlgError("SVD did not converge"); def _raise_linalgerror_lstsq(err, flag): raise LinAlgError("SVD did not converge in Linear Least Squares"); def get_linalg_error_extobj(callback): extobj = list(_linalg_error_extobj); extobj[2] = callback; return extobj; def _makearray(a): new = asarray(a); wrap = getattr(a, "__array_prepare__", new.__array_wrap__); return new, wrap; def isComplexType(t): return issubclass(t, complexfloating); _real_types_map = {single: single,; double: double,; csingle: single,; cdouble: double}; _complex_types_map = {single: csingle,; double: cdouble,; csingle: csingle,; cdouble: cdouble}; def _realType(t, default=double): return _real_types_map.get(t, default); def _complexType(t, default=cdouble): return _complex_types_map.get(t, default); def _linalgRealType(t): """Cast the type t to either double or cdouble."""; return double; def _commonType(*arrays): result_type = single; is_complex = False; for a in arrays: if issubclass(a.dtype.type, inexact): if isComplexType(a.dtype.type): is_complex = True; rt = _realType(a.dtype.type, default=None); if rt is None: raise TypeError("array type %s is unsupported in linalg" %; (a.dtype.name,)); else: rt = double; if rt is double: result_type = double; if is_complex: t = cdouble; result_type = _complex_types_map[result_type]; else: t = double; return t, result_type; _fastCT = fastCopyAndTranspose; def _to_native_byte_order(*arrays): ret = []; for arr in arrays: if arr.dtype.byteorder not in ('=', '|'): ret.append(asarray(arr, dtype=arr.dtype.newbyteorder('='))); else: ret.append(arr); if len(ret) == 1: return ret[0]; else: return ret; def _fastCopyAndTranspose(type, *arrays): cast_arrays = (); for a in arrays: if a.dtype.type is type: cast_arrays = cast_arrays + (_fastCT(a),); else: cast_arrays = cast_arrays + (_fastCT(a.astype(type)),); if len(cast_arrays) == 1: return cast_arrays[0]; else: return cast_arrays; def _assert_2d(*arrays): for a in arrays: if a.ndim != 2: raise LinAlgError('%d-dimensional array given. Array must be '; 'two-dimensional' % a.ndim); def _assert_stacked_2d(*arrays): for a in arrays: if a.ndim < 2: raise LinAlgError('%d-dimensional array given. Array must be '; 'at least two-dimensional' % a.ndim); def _assert_stacked_square(*arrays): for a in arrays: m, n = a.shape[-2:]; if m != n: raise LinAlgError('Last 2 dimensions of the array must be square'); def _assert_finite(*arrays): for a in arrays: if not isfinite(a).all(): raise LinAlgError("Array must not contain infs or NaNs"); def _is_empty_2d(arr): return arr.size == 0 and product(arr.shape[-2:]) == 0; def transpose(a): a, wrap = _makearray(a); b = asarray(b); an = a.ndim; if axes is not None: allaxes = list(range(0, an)); for k in axes: allaxes.remove(k); allaxes.insert(an, k); a = a.transpose(allaxes); oldshape = a.shape[-(an-b.ndim):]; prod = 1; for k in oldshape: prod *= k; a = a.reshape(-1, prod); b = b.ravel(); res = wrap(solve(a, b)); res.shape = oldshape; return res; def _solve_dispatcher(a, b): return (a, b); @array_function_dispatch(_solve_dispatcher); def solve(a, b): a, _ = _makearray(a); _assert_stacked_2d(a); _assert_stacked_square(a); b, wrap = _makearray(b); t, result_t = _commonType(a, b); if b.ndim == a.ndim - 1: gufunc = _umath_linalg.solve1; else: gufunc = _umath_linalg.solve; signature = 'DD->D' if isComplexType(t) else 'dd->d'; extobj = get_linalg_error_extobj(_raise_linalgerror_singular); r = gufunc(a, b, signature=signature, extobj=extobj); return wrap(r.astype(result_t, copy=False)); def _tensorinv_dispatcher(a, ind=None): return (a,); @array_function_dispatch(_tensorinv_dispatcher); def tensorinv(a, ind=2): a = asarray(a); oldshape = a.shape; prod = 1; if ind > 0: invshape = oldshape[ind:] + oldshape[:ind]; for k in oldshape[ind:]: prod *= k; else: raise ValueError("Invalid ind argument."); a = a.reshape(prod, -1); ia = inv(a); return ia.reshape(*invshape); def _unary_dispatcher(a): return (a,); @array_function_dispatch(_unary_dispatcher); def inv(a): a, wrap = _makearray(a); _assert_stacked_2d(a); _assert_stacked_square(a); t, result_t = _commonType(a); signature = 'D->D' if isComplexType(t) else 'd->d'; extobj = get_linalg_error_extobj(_raise_linalgerror_singular); ainv = _umath_linalg.inv(a, signature=signature, extobj=extobj); return wrap(ainv.astype(result_t, copy=False)); def _matrix_power_dispatcher(a, n): return (a,); @array_function_dispatch(_matrix_power_dispatcher); def matrix_power(a, n): a = asanyarray(a); _assert_stacked_2d(a); _assert_stacked_square(a); try: n = operator.index(n); except TypeError: raise TypeError("exponent must be an integer"); if a.dtype != object: fmatmul = matmul; elif a.ndim == 2: fmatmul = dot; else: raise NotImplementedError(; "matrix_power not supported for stacks of object arrays"); if n == 0: a = empty_like(a); a[...] = eye(a.shape[-2], dtype=a.dtype); return a; elif n < 0: a = inv(a); n = abs(n); if n == 1: return a; elif n == 2: return fmatmul(a, a); elif n == 3: return fmatmul(fmatmul(a, a), a); z = result = None; while n > 0: z = a if z is None else fmatmul(z, z); n, bit = divmod(n, 2); if bit: result = z if result is None else fmatmul(result, z); return result; @array_function_dispatch(_unary_dispatcher); def cholesky(a): extobj = get_linalg_error_extobj(_raise_linalgerror_nonposdef); gufunc = _umath_linalg.cholesky_lo; a, wrap = _makearray(a); _assert_stacked_2d(a); _assert_stacked_square(a); t, result_t = _commonType(a); signature = 'D->D' if isComplexType(t) else 'd->d'; r = gufunc(a, signature=signature, extobj=extobj); return wrap(r.astype(result_t, copy=False)); def _qr_dispatcher(a, mode=None): return (a,); @array_function_dispatch(_qr_dispatcher); def qr(a, mode='reduced'): if mode not in ('reduced', 'complete',x 'r', 'raw'): if mode in ('f', 'full'): msg = "".join((; "The 'full' option is deprecated in favor of 'reduced'.\n",; "For backward compatibility let mode default.")); warnings.warn(msg, DeprecationWarning, stacklevel=3); mode = 'reduced'; elif mode in ('e', 'economic'): msg = "The 'economic' option is deprecated."; warnings.warn(msg, DeprecationWarning, stacklevel=3); mode = 'economic'; else: raise ValueError("Unrecognized mode '%s'" % mode); a, wrap = _makearray(a); _assert_2d(a); m, n = a.shape; t, result_t = _commonType(a); a = _fastCopyAndTranspose(t, a); a = _to_native_byte_order(a); mn = min(m, n); tau = zeros((mn,), t); if isComplexType(t): lapack_routine = lapack_lite.zgeqrf; routine_name = 'zgeqrf'; else: lapack_routine = lapack_lite.dgeqrf; routine_name = 'dgeqrf'; lwork = 1; work = zeros((lwork,), t); results = lapack_routine(m, n, a, max(1, m), tau, work, -1, 0); if results['info'] != 0: raise LinAlgError('%s returns %d' % (routine_name, results['info'])); lwork = max(1, n, int(abs(work[0]))); work = zeros((lwork,), t); results = lapack_routine(m, n, a, max(1, m), tau, work, lwork, 0); if results['info'] != 0: raise LinAlgError('%s returns %d' % (routine_name, results['info'])); if mode == 'r': r = _fastCopyAndTranspose(result_t, a[:, :mn]); return wrap(triu(r)); if mode == 'raw': return a, tau; if mode == 'economic': if t != result_t : a = a.astype(result_t, copy=False); return wrap(a.T); if mode == 'complete' and m > n: mc = m; q = empty((m, m), t); else: mc = mn; q = empty((n, m), t); q[:n] = a; if isComplexType(t): lapack_routine = lapack_lite.zungqr; routine_name = 'zungqr'; else: lapack_routine = lapack_lite.dorgqr; routine_name = 'dorgqr'; lwork = 1; work = zeros((lwork,), t); results = lapack_routine(m, mc, mn, q, max(1, m), tau, work, -1, 0); if results['info'] != 0: raise LinAlgError('%s returns %d' % (routine_name, results['info'])); lwork = max(1, n, int(abs(work[0]))); work = zeros((lwork,), t); results = lapack_routine(m, mc, mn, q, max(1, m), tau, work, lwork, 0); if results['info'] != 0: raise LinAlgError('%s returns %d' % (routine_name, results['info'])); q = _fastCopyAndTranspose(result_t, q[:mc]); r = _fastCopyAndTranspose(result_t, a[:, :mc]); return wrap(q), wrap(triu(r)); @array_function_dispatch(_unary_dispatcher); def eigvals(a): a, wrap = _makearray(a); _assert_stacked_2d(a); _assert_stacked_square(a); _assert_finite(a); t, result_t = _commonType(a); extobj = get_linalg_error_extobj(; _raise_linalgerror_eigenvalues_nonconvergence); signature = 'D->D' if isComplexType(t) else 'd->D'; w = _umath_linalg.eigvals(a, signature=signature, extobj=extobj); if not isComplexType(t): if all(w.imag == 0): w = w.real; result_t = _realType(result_t); else: result_t = _complexType(result_t); return w.astype(result_t, copy=False); def _eigvalsh_dispatcher(a, UPLO=None): return (a,); @array_function_dispatch(_eigvalsh_dispatcher); def eigvalsh(a, UPLO='L'): UPLO = UPLO.upper(); if UPLO not in ('L', 'U'): raise ValueError("UPLO argument must be 'L' or 'U'"); extobj = get_linalg_error_extobj(; _raise_linalgerror_eigenvalues_nonconvergence); if UPLO == 'L': gufunc = _umath_linalg.eigvalsh_lo; else: gufunc = _umath_linalg.eigvalsh_up; a, wrap = _makearray(a); _assert_stacked_2d(a); _assert_stacked_square(a); t, result_t = _commonType(a); signature = 'D->d' if isComplexType(t) else 'd->d'; w = gufunc(a, signature=signature, extobj=extobj); return w.astype(_realType(result_t), copy=False); def _convertarray(a): t, result_t = _commonType(a); a = _fastCT(a.astype(t)); return a, t, result_t; def eig(a): a, wrap = _makearray(a); _assert_stacked_2d(a); _assert_stacked_square(a); _assert_finite(a); t, result_t = _commonType(a); extobj = get_linalg_error_extobj(; _raise_linalgerror_eigenvalues_nonconvergence); signature = 'D->DD' if isComplexType(t) else 'd->DD'; w, vt = _umath_linalg.eig(a, signature=signature, extobj=extobj); if not isComplexType(t) and all(w.imag == 0.0): w = w.real; vt = vt.real; result_t = _realType(result_t); else: result_t = _complexType(result_t); vt = vt.astype(result_t, copy=False); return w.astype(result_t, copy=False), wrap(vt); @array_function_dispatch(_eigvalsh_dispatcher); def eigh(a, UPLO='L'): UPLO = UPLO.upper(); if UPLO not in ('L', 'U'): raise ValueError("UPLO argument must be 'L' or 'U'"); a, wrap = _makearray(a); _assert_stacked_2d(a); _assert_stacked_square(a); t, result_t = _commonType(a); extobj = get_linalg_error_extobj(; _raise_linalgerror_eigenvalues_nonconvergence); if UPLO == 'L': gufunc = _umath_linalg.eigh_lo; else: gufunc = _umath_linalg.eigh_up; signature = 'D->dD' if isComplexType(t) else 'd->dd'; w, vt = gufunc(a, signature=signature, extobj=extobj); w = w.astype(_realType(result_t), copy=False); vt = vt.astype(result_t, copy=False); return w, wrap(vt); def _svd_dispatcher(a, full_matrices=None, compute_uv=None, hermitian=None): return (a,); @array_function_dispatch(_svd_dispatcher); def svd(a, full_matrices=True, compute_uv=True, hermitian=False): a, wrap = _makearray(a); if hermitian: if compute_uv: s, u = eigh(a); s = s[..., ::-1]; u = u[..., ::-1]; vt = transpose(u * sign(s)[..., None, :]).conjugate(); s = abs(s); return wrap(u), s, wrap(vt); else: s = eigvalsh(a); s = s[..., ::-1]; s = abs(s); return s; _assert_stacked_2d(a); t, result_t = _commonType(a); extobj = get_linalg_error_extobj(_raise_linalgerror_svd_nonconvergence); m, n = a.shape[-2:]; if compute_uv: if full_matrices: if m < n: gufunc = _umath_linalg.svd_m_f; else: gufunc = _umath_linalg.svd_n_f; else: if m < n: gufunc = _umath_linalg.svd_m_s; else: gufunc = _umath_linalg.svd_n_s; signature = 'D->DdD' if isComplexType(t) else 'd->ddd'; u, s, vh = gufunc(a, signature=signature, extobj=extobj); u = u.astype(result_t, copy=False); s = s.astype(_realType(result_t), copy=False); vh = vh.astype(result_t, copy=False); return wrap(u), s, wrap(vh); else: if m < n: gufunc = _umath_linalg.svd_m; else: gufunc = _umath_linalg.svd_n; signature = 'D->d' if isComplexType(t) else 'd->d'; s = gufunc(a, signature=signature, extobj=extobj); s = s.astype(_realType(result_t), copy=False); return s; def _cond_dispatcher(x, p=None): return (x,); @array_function_dispatch(_cond_dispatcher); def cond(x, p=None): x = asarray(x); if _is_empty_2d(x): raise LinAlgError("cond is not defined on empty arrays"); if p is None or p == 2 or p == -2: s = svd(x, compute_uv=False); with errstate(all='ignore'): if p == -2: r = s[..., -1] / s[..., 0]; else: r = s[..., 0] / s[..., -1]; else: _assert_stacked_2d(x); _assert_stacked_square(x); t, result_t = _commonType(x); signature = 'D->D' if isComplexType(t) else 'd->d'; with errstate(all='ignore'): invx = _umath_linalg.inv(x, signature=signature); r = norm(x, p, axis=(-2, -1)) * norm(invx, p, axis=(-2, -1)); r = r.astype(result_t, copy=False); r = asarray(r); nan_mask = isnan(r); if nan_mask.any(): nan_mask &= ~isnan(x).any(axis=(-2, -1)); if r.ndim > 0: r[nan_mask] = Inf; elif nan_mask: r[()] = Inf; if r.ndim == 0: r = r[()]; return r; def _matrix_rank_dispatcher(M, tol=None, hermitian=None): return (M,); @array_function_dispatch(_matrix_rank_dispatcher); def matrix_rank(M, tol=None, hermitian=False): M = asarray(M); if M.ndim < 2: return int(not all(M==0)); S = svd(M, compute_uv=False, hermitian=hermitian); if tol is None: tol = S.max(axis=-1, keepdims=True) * max(M.shape[-2:]) * finfo(S.dtype).eps; else: tol = asarray(tol)[..., newaxis]; return count_nonzero(S > tol, axis=-1); def pinv(a, rcond=1e-15, hermitian=False): a, wrap = _makearray(a); rcond = asarray(rcond); if _is_empty_2d(a): m, n = a.shape[-2:]; res = empty(a.shape[:-2] + (n, m), dtype=a.dtype); return wrap(res); a = a.conjugate(); u, s, vt = svd(a, full_matrices=False, hermitian=hermitian); cutoff = rcond[..., newaxis] * amax(s, axis=-1, keepdims=True); large = s > cutoff; s = divide(1, s, where=large, out=s); s[~large] = 0; res = matmul(transpose(vt), multiply(s[..., newaxis], transpose(u))); return wrap(res); def slogdet(a): a = asarray(a); _assert_stacked_2d(a); _assert_stacked_square(a); t, result_t = _commonType(a); real_t = _realType(result_t); signature = 'D->Dd' if isComplexType(t) else 'd->dd'; sign, logdet = _umath_linalg.slogdet(a, signature=signature); sign = sign.astype(result_t, copy=False); logdet = logdet.astype(real_t, copy=False); return sign, logdet; @array_function_dispatch(_unary_dispatcher); def det(a): a = asarray(a); _assert_stacked_2d(a); _assert_stacked_square(a); t, result_t = _commonType(a); signature = 'D->D' if isComplexType(t) else 'd->d'; r = _umath_linalg.det(a, signature=signature); r = r.astype(result_t, copy=False); return r; def lstsq(a, b, rcond="warn"): a, _ = _makearray(a); b, wrap = _makearray(b); is_1d = b.ndim == 1; if is_1d: b = b[:, newaxis]; _assert_2d(a, b); m, n = a.shape[-2:]; m2, n_rhs = b.shape[-2:]; if m != m2: raise LinAlgError('Incompatible dimensions'); t, result_t = _commonType(a, b); real_t = _linalgRealType(t); result_real_t = _realType(result_t); if rcond == "warn": warnings.warn("`rcond` parameter will change to the default of "; "machine precision times ``max(M, N)`` where M and N "; "are the input matrix dimensions.\n"; "To use the future default and silence this warning "; "we advise to pass `rcond=None`, to keep using the old, "; "explicitly pass `rcond=-1`.",; FutureWarning, stacklevel=3); rcond = -1; if rcond is None: rcond = finfo(t).eps * max(n, m); if m <= n: gufunc = _umath_linalg.lstsq_m; else: gufunc = _umath_linalg.lstsq_n; signature = 'DDd->Ddid' if isComplexType(t) else 'ddd->ddid'; extobj = get_linalg_error_extobj(_raise_linalgerror_lstsq); if n_rhs == 0: b = zeros(b.shape[:-2] + (m, n_rhs + 1), dtype=b.dtype); x, resids, rank, s = gufunc(a, b, rcond, signature=signature, extobj=extobj); if m == 0: x[...] = 0; if n_rhs == 0: x = x[..., :n_rhs]; resids = resids[..., :n_rhs]; if is_1d: x = x.squeeze(axis=-1); if rank != n or m <= n: resids = array([], result_real_t); s = s.astype(result_real_t, copy=False); resids = resids.astype(result_real_t, copy=False); x = x.astype(result_t, copy=True); return wrap(x), wrap(resids), rank, s; def _multi_svd_norm(x, row_axis, col_axis, op): y = moveaxis(x, (row_axis, col_axis), (-2, -1)); result = op(svd(y, compute_uv=False), axis=-1); return result; def _norm_dispatcher(x, ord=None, axis=None, keepdims=None): return (x,); @array_function_dispatch(_norm_dispatcher); def norm(x, ord=None, axis=None, keepdims=False): x = asarray(x); if not issubclass(x.dtype.type, (inexact, object_)): x = x.astype(float); if axis is None: ndim = x.ndim; if ((ord is None) or; (ord in ('f', 'fro') and ndim == 2) or; (ord == 2 and ndim == 1)): x = x.ravel(order='K'); if isComplexType(x.dtype.type): sqnorm = dot(x.real, x.real) + dot(x.imag, x.imag); else: sqnorm = dot(x, x); ret = sqrt(sqnorm); if keepdims: ret = ret.reshape(ndim*[1]); return ret; nd = x.ndim; if axis is None: axis = tuple(range(nd)); elif not isinstance(axis, tuple): try: axis = int(axis); except Exception: raise TypeError("'axis' must be None, an integer or a tuple of integers"); axis = (axis,); if len(axis) == 1: if ord == Inf: return abs(x).max(axis=axis, keepdims=keepdims); elif ord == -Inf: return abs(x).min(axis=axis, keepdims=keepdims); elif ord == 0: return (x != 0).astype(x.real.dtype).sum(axis=axis, keepdims=keepdims); elif ord == 1: return add.reduce(abs(x), axis=axis, keepdims=keepdims); elif ord is None or ord == 2: s = (x.conj() * x).real; return sqrt(add.reduce(s, axis=axis, keepdims=keepdims)); else: try: ord + 1; except TypeError: raise ValueError("Invalid norm order for vectors."); absx = abs(x); absx **= ord; ret = add.reduce(absx, axis=axis, keepdims=keepdims); ret **= (1 / ord); return ret; elif len(axis) == 2: row_axis, col_axis = axis; row_axis = normalize_axis_index(row_axis, nd); col_axis = normalize_axis_index(col_axis, nd); if row_axis == col_axis: raise ValueError('Duplicate axes given.'); if ord == 2: ret =_multi_svd_norm(x, row_axis, col_axis, amax); elif ord == -2: ret = _multi_svd_norm(x, row_axis, col_axis, amin); elif ord == 1: if col_axis > row_axis: col_axis -= 1; ret = add.reduce(abs(x), axis=row_axis).max(axis=col_axis); elif ord == Inf: if row_axis > col_axis: row_axis -= 1; ret = add.reduce(abs(x), axis=col_axis).max(axis=row_axis); elif ord == -1: if col_axis > row_axis: col_axis -= 1; ret = add.reduce(abs(x), axis=row_axis).min(axis=col_axis); elif ord == -Inf: if row_axis > col_axis: row_axis -= 1; ret = add.reduce(abs(x), axis=col_axis).min(axis=row_axis); elif ord in [None, 'fro', 'f']: ret = sqrt(add.reduce((x.conj() * x).real, axis=axis)); elif ord == 'nuc': ret = _multi_svd_norm(x, row_axis, col_axis, sum); else: raise ValueError("Invalid norm order for matrices."); if keepdims: ret_shape = list(x.shape); ret_shape[axis[0]] = 1; ret_shape[axis[1]] = 1; ret = ret.reshape(ret_shape); return ret; else: raise ValueError("Improper number of dimensions to norm."); def multi_dot(arrays): n = len(arrays); if n < 2: raise ValueError("Expecting at least two arrays."); elif n == 2: return dot(arrays[0], arrays[1]); arrays = [asanyarray(a) for a in arrays]; ndim_first, ndim_last = arrays[0].ndim, arrays[-1].ndim; if arrays[0].ndim == 1: arrays[0] = atleast_2d(arrays[0]); if arrays[-1].ndim == 1: arrays[-1] = atleast_2d(arrays[-1]).T; _assert_2d(*arrays) if n == 3: result = _multi_dot_three(arrays[0], arrays[1], arrays[2]) else: order = _multi_dot_matrix_chain_order(arrays) result = _multi_dot(arrays, order, 0, n - 1) if ndim_first == 1 and ndim_last == 1: return result[0, 0] elif ndim_first == 1 or ndim_last == 1: return result.ravel() else: return result; def _multi_dot_three(A, B, C): a0, a1b0 = A.shape b1c0, c1 = C.shape; cost1 = a0 * b1c0 * (a1b0 + c1); cost2 = a1b0 * c1 * (a0 + b1c0); if cost1 < cost2: return dot(dot(A, B), C); else: return dot(A, dot(B, C)); def _multi_dot_matrix_chain_order(arrays, return_costs=False): n = len(arrays); p = [a.shape[0] for a in arrays] + [arrays[-1].shape[1]]; m = zeros((n, n), dtype=double); s = empty((n, n), dtype=intp); for l in range(1, n): for i in range(n - l): j = i + l; m[i, j] = Inf; for k in range(i, j): q = m[i, k] + m[k+1, j] + p[i]*p[k+1]*p[j+1]; if q < m[i, j]: m[i, j] = q; s[i, j] = k; if i == j: return arrays[i]; else: return dot(_multi_dot(arrays, order, i, order[i, j]),; _multi_dot(arrays, order, order[i, j] + 1, j))
KameleonWindowLearnScale from kameleon_mcmc.mcmc.samplers.StandardMetropolis import StandardMetropolis if __name__ == '__main__': experiment_dir = str(os.path.abspath(sys.argv[0])).split(os.sep)[-1].split(".")[0] + os.sep distribution = Flower(amplitude=6, frequency=6, variance=1, radius=10, dimension=8) sigma = 5 kernel = GaussianKernel(sigma=sigma) burnin = 60000 num_iterations = 120000 #mcmc_sampler = KameleonWindowLearnScale(distribution, kernel, stop_adapt=burnin) mean_est = zeros(distribution.dimension, dtype="float64") cov_est = 1.0 * eye(distribution.dimension) #mcmc_sampler = AdaptiveMetropolisLearnScale(distribution, mean_est=mean_est, cov_est=cov_est) #mcmc_sampler = AdaptiveMetropolis(distribution, mean_est=mean_est, cov_est=cov_est) mcmc_sampler = StandardMetropolis(distribution) start = zeros(distribution.dimension, dtype="float64") mcmc_params = MCMCParams(start=start, num_iterations=num_iterations, burnin=burnin) mcmc_chain = MCMCChain(mcmc_sampler, mcmc_params) mcmc_chain.append_mcmc_output(StatisticsOutput()) experiment = SingleChainExperiment(mcmc_chain, experiment_dir) experiment.run()
from distribution.Gaussian import Gaussian from distribution.SampleObject import SampleObject from mcmc.chain.Chain import Chain from mcmc.chain.ChainParams import ChainParams from mcmc.sampler.MetropolisHastings import MetropolisHastings from numpy.lib.twodim_base import eye from numpy.ma.core import zeros if __name__ == '__main__': start=SampleObject(zeros(2)) target=Gaussian(mu=zeros(2), Sigma=eye(2)) sampler=MetropolisHastings(target, start) params=ChainParams(1000) chain=Chain(params, sampler) chain.run()
# throw away some data n = 250 seed(1) idx = permutation(len(data)) idx = idx[:n] data = data[idx] labels = labels[idx] # normalise and whiten dataset data -= mean(data, 0) L = cholesky(cov(data.T)) data = solve_triangular(L, data.T, lower=True).T dim = shape(data)[1] # prior on theta and posterior target estimate theta_prior = Gaussian(mu=0 * ones(dim), Sigma=eye(dim) * 5) target=PseudoMarginalHyperparameterDistribution(data, labels, \ n_importance=100, prior=theta_prior, \ ridge=1e-3) # create sampler burnin = 10000 num_iterations = burnin + 300000 kernel = GaussianKernel(sigma=23.0) sampler = KameleonWindowLearnScale(target, kernel, stop_adapt=burnin) # sampler=AdaptiveMetropolisLearnScale(target) # sampler=StandardMetropolis(target) # posterior mode derived by initial tests start = zeros(target.dimension) params = MCMCParams(start=start,
# loop over parameters here experiment_dir=experiment_dir_base +os.sep + str(os.path.abspath(sys.argv[0])).split(os.sep)[-1].split(".")[0] + os.sep print "running experiments", n, "times at base", experiment_dir for i in range(n): distribution = Ring() mcmc_samplers = [] kernel = GaussianKernel(sigma=1) # mcmc_samplers.append(KameleonWindow(distribution, kernel)) mcmc_samplers.append(KameleonWindowLearnScale(distribution, kernel)) mean_est = array([-2.0, -2.0]) cov_est = 0.05 * eye(2) # mcmc_samplers.append(AdaptiveMetropolis(distribution, mean_est=mean_est, cov_est=cov_est)) mcmc_samplers.append(AdaptiveMetropolisLearnScale(distribution, mean_est=mean_est, cov_est=cov_est)) num_eigen = 2 mcmc_samplers.append(AdaptiveMetropolisPCA(distribution, num_eigen=num_eigen, mean_est=mean_est, cov_est=cov_est)) mcmc_samplers.append(StandardMetropolis(distribution)) start = array([-2.0, -2.0]) mcmc_params = MCMCParams(start=start, num_iterations=num_iterations, burnin=burnin) mcmc_chains = [MCMCChain(mcmc_sampler, mcmc_params) for mcmc_sampler in mcmc_samplers] for mcmc_chain in mcmc_chains: mcmc_chain.append_mcmc_output(StatisticsOutput())
def precompute(self): # collect lines for Graphlab graph definition file for full rank case graphlab_lines=GraphlabLines(output_filename=self.output_filename) # compute all non-symmetric kernels for incoming messages at a node print "precomputing (non-symmetric) kernels for incoming messages at a node" graphlab_lines.lines.append("# non-observed nodes") for node in self.graph: added_node=False for in_message in self.graph[node]: for out_message in self.graph[node]: if in_message==out_message: continue # dont add nodes which have no kernels, and only do once if they have if not added_node: graphlab_lines.new_non_observed_node(node) added_node=True edge_in_message=(node, in_message) edge_out_message=(out_message, node) lhs=self.data[edge_in_message][0] rhs=self.data[edge_out_message][1] lhs=reshape(lhs, (len(lhs),1)) rhs=reshape(rhs, (len(rhs),1)) K=self.kernel.kernel(lhs,rhs) graphlab_lines.add_non_observed_node(node, out_message, in_message, K) print "precomputing kernel (vectors) at observed nodes" graphlab_lines.lines.append(os.linesep + "# observed nodes") for node, observation in self.observations.items(): graphlab_lines.new_observed_node(node) for out_message in self.graph[node]: edge=(out_message, node) lhs=self.data[edge][1] lhs=reshape(lhs, (len(lhs), 1)) rhs=[[observation]] K=self.kernel.kernel(lhs, rhs) graphlab_lines.add_observed_node(node, out_message, K) # now precompute systems for inference print "precomputing systems for messages from observed nodes" graphlab_lines.lines.append(os.linesep + "# edges with observed targets") for node, observation in self.observations.items(): for out_message in self.graph[node]: edge=(out_message, node) graphlab_lines.new_edge_observed_target(node, out_message) data_source=self.data[edge][0] data_source=reshape(data_source, (len(data_source), 1)) data_target=self.data[edge][1] data_target=reshape(data_target, (len(data_target), 1)) Ks=self.kernel.kernel(data_source) Kt=self.kernel.kernel(data_target) Ls=cholesky(Ks+eye(shape(Ks)[0])*self.reg_lambda) Lt=cholesky(Kt+eye(shape(Kt)[0])*self.reg_lambda) graphlab_lines.add_edge(node, out_message,"L_s", Ls) graphlab_lines.add_edge(node, out_message,"L_t", Lt) print "precomputing systems for messages from non-observed nodes" graphlab_lines.lines.append(os.linesep + "# edges with non-observed targets") for edge in self.edges: # exclude edges which involve observed nodes is_edge_target_observed=len(Set(self.observations.keys()).intersection(Set(edge)))>0 if not is_edge_target_observed: graphlab_lines.new_edge_observed_target(edge[1], edge[0]) data_source=self.data[edge][0] data_source=reshape(data_source, (len(data_source), 1)) Ks=self.kernel.kernel(data_source) Ls=cholesky(Ks+eye(shape(Ks)[0])*self.reg_lambda) graphlab_lines.add_edge(edge[1], edge[0],"L_s", Ls) # write graph definition file to disc graphlab_lines.flush()
# loop over parameters here experiment_dir = experiment_dir_base + str(os.path.abspath(sys.argv[0])).split(os.sep)[-1].split(".")[0] + os.sep print "running experiments", n, "times at base", experiment_dir # load data data,labels=GPData.get_glass_data() # normalise and whiten dataset data-=mean(data, 0) L=cholesky(cov(data.T)) data=solve_triangular(L, data.T, lower=True).T dim=shape(data)[1] # prior on theta and posterior target estimate theta_prior=Gaussian(mu=0*ones(dim), Sigma=eye(dim)*5) distribution=PseudoMarginalHyperparameterDistribution(data, labels, \ n_importance=100, prior=theta_prior, \ ridge=1e-3) sigma = 23.0 print "using sigma", sigma kernel = GaussianKernel(sigma=sigma) for i in range(n): mcmc_samplers = [] burnin=20000 num_iterations=100000
distribution = Banana(dimension=8, bananicity=0.03, V=100) sigma = GaussianKernel.get_sigma_median_heuristic( distribution.sample(1000).samples) sigma = 10 print "using sigma", sigma kernel = GaussianKernel(sigma=sigma) burnin = 20000 num_iterations = 40000 mcmc_sampler = KameleonWindowLearnScale(distribution, kernel, stop_adapt=burnin) mean_est = zeros(distribution.dimension, dtype="float64") cov_est = 1.0 * eye(distribution.dimension) cov_est[0, 0] = distribution.V #mcmc_sampler = AdaptiveMetropolisLearnScale(distribution, mean_est=mean_est, cov_est=cov_est) #mcmc_sampler = AdaptiveMetropolis(distribution, mean_est=mean_est, cov_est=cov_est) #mcmc_sampler = StandardMetropolis(distribution) start = zeros(distribution.dimension, dtype="float64") mcmc_params = MCMCParams(start=start, num_iterations=num_iterations, burnin=burnin) mcmc_chain = MCMCChain(mcmc_sampler, mcmc_params) mcmc_chain.append_mcmc_output(StatisticsOutput()) experiment = SingleChainExperiment(mcmc_chain, experiment_dir) experiment.run()
def __init__(self, mu=array([0, 0]), Sigma=eye(2), is_cholesky=False, ell=None): Distribution.__init__(self, len(Sigma)) assert(len(shape(mu)) == 1) assert(max(shape(Sigma)) == len(mu)) self.mu = mu self.ell = ell if is_cholesky: self.L = Sigma if ell == None: assert(shape(Sigma)[0] == shape(Sigma)[1]) else: assert(shape(Sigma)[1] == ell) else: assert(shape(Sigma)[0] == shape(Sigma)[1]) if ell is not None: self.L, _, _ = MatrixTools.low_rank_approx(Sigma, ell) self.L = self.L.T assert(shape(self.L)[1] == ell) else: try: self.L = cholesky(Sigma) except LinAlgError: # some really crude check for PSD (which only corrects for orunding errors self.L = cholesky(Sigma+eye(len(Sigma))*1e-5)