def setUp(self): super(TestSMCUpdater, self).setUp() self.precession_model = SimplePrecessionModel() self.num_precession_model = NumericalSimplePrecessionModel() self.expparams = TestSMCUpdater.TEST_EXPPARAMS.reshape(-1, 1) self.outcomes = self.precession_model.simulate_experiment( TestSMCUpdater.MODELPARAMS, TestSMCUpdater.TEST_EXPPARAMS, repeat=1).reshape(-1, 1) self.updater = SMCUpdater(self.precession_model, TestSMCUpdater.N_PARTICLES, TestSMCUpdater.PRIOR) self.updater_bayes = SMCUpdaterBCRB(self.precession_model, TestSMCUpdater.N_PARTICLES, TestSMCUpdater.PRIOR, adaptive=True) self.num_updater = SMCUpdater(self.num_precession_model, TestSMCUpdater.N_PARTICLES, TestSMCUpdater.PRIOR) self.num_updater_bayes = SMCUpdaterBCRB(self.num_precession_model, TestSMCUpdater.N_PARTICLES, TestSMCUpdater.PRIOR, adaptive=True)
def test_region_est_ellipsoid(self): """ Tests that region_est_ellipsoid works. """ dist = MultivariateNormalDistribution(self.MEAN, self.COV) # the model is irrelevant; we just want the updater to have some particles # with the desired normal distribution. u = SMCUpdater(MockModel(4), self.N_PARTICLES, dist) # ask for a confidence level of 0.5 A, c = u.region_est_ellipsoid(level=0.5) # center of ellipse should be the mean of the multinormal assert_almost_equal(np.round(c), self.MEAN, 1) # finally, the principal lengths of the ellipsoid # should be the same as COV _, QA, _ = np.linalg.svd(A) _, QC, _ = np.linalg.svd(self.COV) QA, QC = np.sqrt(QA), np.sqrt(QC) assert_almost_equal( QA / np.linalg.norm(QA), QC / np.linalg.norm(QC), 1 )
def main(): m = SimpleMacroModel() prior = UniformDistribution([[0, 1], [0, 1]]) u = SMCUpdater(m, 1000, prior) modelparams = prior.sample() expparams = np.array([(12.0, )], dtype=m.expparams_dtype) datum = m.simulate_experiment(modelparams, expparams) print datum u.update(datum, expparams) print u.est_mean() print m.call_count
def setUp(self): super(TestBayesRisk, self).setUp() # Set up relevant models. self.coin_model = CoinModel() self.binomial_model = BinomialModel(self.coin_model) # Set up updaters for these models using particle approximations # of conjugate priors self.updater_binomial = SMCUpdater(self.binomial_model, TestBayesRisk.N_PARTICLES, TestBayesRisk.PRIOR_BETA)
def setUp(self): super(TestFisherInformation, self).setUp() # Set up relevant models. self.coin_model = CoinModel() self.binomial_model = DifferentiableBinomialModel(self.coin_model) # Set up updaters for these models using particle approximations # of conjugate priors self.updater_binomial = SMCUpdater(self.binomial_model, TestFisherInformation.N_PARTICLES, TestFisherInformation.PRIOR_BETA)
def do_update(model, n_particles, prior, outcomes, expparams, return_all, resampler=None): updater = SMCUpdater(model, n_particles, prior, resampler=resampler ) updater.batch_update(outcomes, expparams, resample_interval=1) mean = updater.est_mean() cov = updater.est_covariance_mtx() if model.n_modelparams == 1: mean = mean[0] cov = cov[0, 0] if not return_all: return mean, cov else: return mean, cov, { 'updater': updater }
def test_region_est_hull(self): """ Tests that test_region_est_hull works """ dist = MultivariateNormalDistribution(self.MEAN, self.COV) # the model is irrelevant; we just want the updater to have some particles # with the desired normal distribution. u = SMCUpdater(MockModel(self.N_MPS), self.N_PARTICLES, dist) faces, vertices = u.region_est_hull(level=0.95) # In this multinormal case, the convex hull surface # should be centered at MEAN assert_almost_equal(np.round(np.mean(vertices, axis=0)), np.round(self.MEAN)) # And a lower level should result in a smaller hull # and therefore smaller sample variance faces2, vertices2 = u.region_est_hull(level=0.2) assert_array_less(np.var(vertices2, axis=0), np.var(vertices, axis=0))
def test_est_credible_region(self): """ Tests that est_credible_region doesn't fail miserably """ dist = MultivariateNormalDistribution(self.MEAN, self.COV) # the model is irrelevant; we just want the updater to have some particles # with the desired normal distribution. u = SMCUpdater(MockModel(self.N_MPS), self.N_PARTICLES, dist) # first check that 0.95 confidence points consume 0.9 confidence points points1 = u.est_credible_region(level=0.95) points2 = u.est_credible_region(level=0.9) assert_almost_equal( np.sort(unique_rows(np.concatenate([points1, points2])), axis=0), np.sort(points1, axis=0)) # do the same thing with different slice points1 = u.est_credible_region(level=0.95, modelparam_slice=self.SLICE) points2 = u.est_credible_region(level=0.9, modelparam_slice=self.SLICE) assert_almost_equal( np.sort(unique_rows(np.concatenate([points1, points2])), axis=0), np.sort(points1, axis=0))
def _mk_updater(self, n_particles, **kwargs): return SMCUpdater(self.model, n_particles, UniformDistribution([0, 1]), **kwargs)
mps_buf.release() eps_buf.release() dest_buf.release() # Now we concatenate over outcomes. return FiniteOutcomeModel.pr0_to_likelihood_array(outcomes, pr0) ## SCRIPT ###################################################################### if __name__ == "__main__": # NOTE: This is now redundant with the perf_testing module. simple_model = SimplePrecessionModel() for model in [AcceleratedPrecessionModel(), SimplePrecessionModel()]: true = np.random.random(1) updater = SMCUpdater(model, 100000, UniformDistribution([0, 1])) tic = time.time() for idx_exp in range(200): if not (idx_exp % 20): print(idx_exp) expparams = np.array([(9 / 8)**idx_exp]) updater.update(simple_model.simulate_experiment(true, expparams), expparams) print(model, updater.est_mean(), true, time.time() - tic)
def test_in_credible_region(self): """ Tests that in_credible_region works. """ dist = MultivariateNormalDistribution(self.MEAN, self.COV) # the model is irrelevant; we just want the updater to have some particles # with the desired normal distribution. u = SMCUpdater(MockModel(4), self.N_PARTICLES, dist) # some points to test with test_points = np.random.multivariate_normal(self.MEAN, self.COV, self.N_PARTICLES) # method='pce' results = [ u.in_credible_region(test_points, level=0.9, method='pce'), u.in_credible_region(test_points, level=0.84, method='pce'), u.in_credible_region(test_points, level=0.5, method='pce'), ] assert_almost_equal( np.array([np.mean(x.astype('float')) for x in results]), np.array([0.9, 0.84, 0.5]), 3 ) # method='hpd-hull' results1 = [ u.in_credible_region(test_points, level=0.9, method='hpd-hull'), u.in_credible_region(test_points, level=0.84, method='hpd-hull'), u.in_credible_region(test_points, level=0.5, method='hpd-hull'), ] assert_array_less( np.array([0.9, 0.84, 0.5]), np.array([np.mean(x.astype('float')) for x in results1]) ) # method='hpd-mvee' results2 = [ u.in_credible_region(test_points, level=0.9, method='hpd-mvee'), u.in_credible_region(test_points, level=0.84, method='hpd-mvee'), u.in_credible_region(test_points, level=0.5, method='hpd-mvee'), ] assert_array_less( np.array([0.9, 0.84, 0.5]), np.array([np.mean(x.astype('float')) for x in results2]) ) # the mvee should be bigger than the convex hull. # this passes iff all points in the ellipses are # also in the hulls. assert_array_less( np.hstack([x.astype('float') for x in results1]), np.hstack([x.astype('float') for x in results2]) + 0.5 ) # check for no failures with slices. u.in_credible_region(test_points[:100,self.SLICE], level=0.9, method='pce', modelparam_slice=self.SLICE) u.in_credible_region(test_points[:100,self.SLICE], level=0.9, method='hpd-hull', modelparam_slice=self.SLICE) u.in_credible_region(test_points[:100,self.SLICE], level=0.9, method='hpd-mvee', modelparam_slice=self.SLICE) # check for no failures with single inputs assert(u.in_credible_region(test_points[0,:], level=0.9, method='pce').size == 1) assert(u.in_credible_region(test_points[0,:], level=0.9, method='hpd-hull').size == 1) assert(u.in_credible_region(test_points[0,:], level=0.9, method='hpd-mvee').size == 1)
def perf_test(model, n_particles, prior, n_exp, heuristic_class, true_model=None, true_prior=None, true_mps=None, extra_updater_args=None): """ Runs a trial of using SMC to estimate the parameters of a model, given a number of particles, a prior distribution and an experiment design heuristic. :param qinfer.Model model: Model whose parameters are to be estimated. :param int n_particles: Number of SMC particles to use. :param qinfer.Distribution prior: Prior to use in selecting SMC particles. :param int n_exp: Number of experimental data points to draw from the model. :param qinfer.Heuristic heuristic_class: Constructor function for the experiment design heuristic to be used. :param qinfer.Model true_model: Model to be used in generating experimental data. If ``None``, assumed to be ``model``. Note that if the true and estimation models have different numbers of parameters, the loss will be calculated by aligning the respective model vectors "at the right," analogously to the convention used by NumPy broadcasting. :param qinfer.Distribution true_prior: Prior to be used in selecting the true model parameters. If ``None``, assumed to be ``prior``. :param numpy.ndarray true_mps: The true model parameters. If ``None``, it will be sampled from ``true_prior``. Note that as this function runs exactly one trial, only one model parameter vector may be passed. In particular, this requires that ``len(true_mps.shape) == 1``. :param dict extra_updater_args: Extra keyword arguments for the updater, such as resampling and zero-weight policies. :rtype np.ndarray: See :ref:`perf_testing_struct` for more details on the type returned by this function. :return: A record array of performance metrics, indexed by the number of experiments performed. """ if true_model is None: true_model = model if true_prior is None: true_prior = prior if true_mps is None: true_mps = true_prior.sample() if extra_updater_args is None: extra_updater_args = {} n_min_modelparams = min(model.n_modelparams, true_model.n_modelparams) dtype, is_scalar_exp = actual_dtype(model, true_model) performance = np.zeros((n_exp, ), dtype=dtype) updater = SMCUpdater(model, n_particles, prior, **extra_updater_args) heuristic = heuristic_class(updater) for idx_exp in range(n_exp): # Set inside the loop to handle the case where the # true model is time-dependent as well as the estimation model. performance[idx_exp]['true'] = true_mps expparams = heuristic() datum = true_model.simulate_experiment(true_mps, expparams) with timing() as t: updater.update(datum, expparams) # Update the true model. true_mps = true_model.update_timestep(promote_dims_left(true_mps, 2), expparams)[:, :, 0] est_mean = updater.est_mean() delta = np.subtract(*shorten_right(est_mean, true_mps)) loss = np.dot(delta**2, model.Q[-n_min_modelparams:]) performance[idx_exp]['elapsed_time'] = t.delta_t performance[idx_exp]['loss'] = loss performance[idx_exp]['resample_count'] = updater.resample_count performance[idx_exp]['outcome'] = datum performance[idx_exp]['est'] = est_mean if is_scalar_exp: performance[idx_exp]['experiment'] = expparams else: for param_name in [param[0] for param in model.expparams_dtype]: performance[idx_exp][param_name] = expparams[param_name] return performance
def perf_test( model, n_particles, prior, n_exp, heuristic_class, true_model=None, true_prior=None, true_mps=None, extra_updater_args=None ): """ Runs a trial of using SMC to estimate the parameters of a model, given a number of particles, a prior distribution and an experiment design heuristic. :param qinfer.Model model: Model whose parameters are to be estimated. :param int n_particles: Number of SMC particles to use. :param qinfer.Distribution prior: Prior to use in selecting SMC particles. :param int n_exp: Number of experimental data points to draw from the model. :param qinfer.Heuristic heuristic_class: Constructor function for the experiment design heuristic to be used. :param qinfer.Model true_model: Model to be used in generating experimental data. If ``None``, assumed to be ``model``. :param qinfer.Distribution true_prior: Prior to be used in selecting the true model parameters. If ``None``, assumed to be ``prior``. :param np.ndarray true_mps: The true model parameters. If ``None``, it will be sampled from ``true_prior``. Note that the performance record can only handle one outcome and therefore ONLY ONE TRUE MODEL. An error will occur if ``true_mps.shape[0] > 1`` returns ``True``. :param dict extra_updater_args: Extra keyword arguments for the updater, such as resampling and zero-weight policies. :rtype np.ndarray: See :ref:`perf_testing_struct` for more details on the type returned by this function. :return: A record array of performance metrics, indexed by the number of experiments performed. """ if true_model is None: true_model = model if true_prior is None: true_prior = prior if true_mps is None: true_mps = true_prior.sample() if extra_updater_args is None: extra_updater_args = {} dtype, is_scalar_exp = actual_dtype(model) performance = np.zeros((n_exp,), dtype=dtype) updater = SMCUpdater(model, n_particles, prior, **extra_updater_args) heuristic = heuristic_class(updater) performance['true'] = true_mps for idx_exp in xrange(n_exp): expparams = heuristic() datum = true_model.simulate_experiment(true_mps, expparams) with timing() as t: updater.update(datum, expparams) est_mean = updater.est_mean() delta = est_mean - true_mps loss = np.dot(delta**2, model.Q) performance[idx_exp]['elapsed_time'] = t.delta_t performance[idx_exp]['loss'] = loss performance[idx_exp]['resample_count'] = updater.resample_count performance[idx_exp]['outcome'] = datum performance[idx_exp]['est'] = est_mean if is_scalar_exp: performance[idx_exp]['experiment'] = expparams else: for param_name in [param[0] for param in model.expparams_dtype]: performance[idx_exp][param_name] = expparams[param_name] return performance
# Model and prior initialization. prior = distributions.HilbertSchmidtUniform() model = QubitStatePauliModel() expparams = np.array( [ ([1, 0, 0], 1), # Records are indicated by tuples. ([0, 1, 0], 1), ([0, 0, 1], 1) ], dtype=model.expparams_dtype) # Make a dict of updater constructors. This will define what kinds # of perfomance data we care about. updater_ctors = dict() if do_smc: updater_ctors['SMC'] = lambda: SMCUpdater(model, n_particles, prior) if do_ale: ale_model = ale.ALEApproximateModel(model, error_tol=err_tol, est_hedge=hedge, adapt_hedge=hedge) updater_ctors['SMC_ALE'] = lambda: SMCUpdater(ale_model, n_particles, prior) # Make a dtype for holding performance data in a record array. # Note that we could do this with out record arrays, but it's easy to # use field names this way. performance_dtype = [ ('est_mean', 'f8'), ('est_cov_mat', 'f8'), ('true_err', 'f8'),