(str(dataset[0].shape), naive_lml, fitted_lml, success),
'test_result')
num_successes += success
assert num_successes == len(self.datasets)
def test_prediction(self):
""" Tests for prediction on a test set.
"""
self.report('Prediction. Probabilistic test, might fail.')
num_successes = 0
for dataset in self.datasets:
naive_gp = build_gp_with_dataset(dataset)
naive_preds, _ = naive_gp.eval(dataset[3])
naive_err = compute_average_prediction_error(dataset, naive_preds)
fitted_gp = fit_gp_with_dataset(dataset)
fitted_preds, _ = fitted_gp.eval(dataset[3])
fitted_err = compute_average_prediction_error(
dataset, fitted_preds)
success = fitted_err <= naive_err
self.report(
'(N,D)=%s:: naive-err=%0.4f, fitted-err=%0.4f, succ=%d' %
(str(dataset[0].shape), naive_err, fitted_err, success),
'test_result')
self.report(' -- GP: %s' % (str(fitted_gp)), 'test_result')
num_successes += success
assert num_successes > 0.6 * len(self.datasets)
if __name__ == '__main__':
execute_tests(5424)
self.report('Test for drawing samples. Probabilistic test, might fail.')
total_coverage = 0
num_test_pts = 100
num_samples = 5 # Draw 5 samples at each point - just for testing.
mfgp_instances = gen_mf_gp_instances()
for inst in mfgp_instances:
Z_test = np.random.random((num_test_pts, inst.fidel_dim))
X_test = np.random.random((num_test_pts, inst.domain_dim))
F_test = inst.post_gp.draw_mf_samples(num_samples, Z_test, X_test)
post_mean, post_std = inst.post_gp.eval_at_fidel(Z_test, X_test, uncert_form='std')
conf_band_width = 1.96
ucb = post_mean + conf_band_width * post_std
lcb = post_mean - conf_band_width * post_std
below_ucb = F_test <= ucb
above_lcb = F_test >= lcb
coverage = (below_ucb * above_lcb).mean()
total_coverage += coverage
self.report(('(n, DZ, DX) = (%d, %d, %d)::: Coverage for 0.95 credible interval: ' +
'%0.4f')%(inst.num_data, inst.fidel_dim, inst.domain_dim, coverage),
'test_result')
avg_coverage = total_coverage / len(mfgp_instances)
avg_coverage_is_good = avg_coverage > 0.9
self.report('Avg coverage (%0.3f) is larger than 0.9? %d'%(avg_coverage,
avg_coverage_is_good), 'test_result')
assert avg_coverage_is_good
if __name__ == '__main__':
execute_tests()
""" Test for the Marginal likelihood of an additive GP. """
self.report(
'Marginal Likelihoods for non-additive vs additive SE GP.' +
' Probabilistic test, might fail.')
self._prediction_test(fit_se_gp_with_dataset,
fit_add_se_gp_with_dataset, 'non-additive',
'additive')
def test_matern_marg_likelihood(self):
""" Test for the Marginal likelihood of an additive GP. """
self.report(
'Marginal Likelihoods for non-additive vs additive Matern GP.' +
' Probabilistic test, might fail.')
self._marg_likelihood_test(fit_matern_gp_with_dataset,
fit_add_matern_gp_with_dataset,
'non-additive', 'additive')
def test_matern_prediction(self):
""" Test for the Marginal likelihood of an additive GP. """
self.report(
'Marginal Likelihoods for non-additive vs additive Matern GP.' +
' Probabilistic test, might fail.')
self._prediction_test(fit_matern_gp_with_dataset,
fit_add_matern_gp_with_dataset, 'non-additive',
'additive')
if __name__ == '__main__':
execute_tests(2349)
# execute_tests()
