class TestDatastoreHistogram(unittest.TestCase):
def setUp(self):
self._ref_entries = [
-9999., -8279., 3.3, 5.5, 2.2, 8.5, 10., 10.2, 10000., 1e7
]
self._ref_n_bins_auto = 25
self._ref_n_bins_manual = 10
self._ref_n_bin_range = (0., 10.)
self._ref_bin_edges_manual_equalspacing = np.linspace(
0, 10, self._ref_n_bins_manual + 1)
# [0 , 1 , 2 , 3 , 4 , 5 , 6 , 7 , 8 , 9 , 10]
self._ref_data_manual_equalspacing = np.array(
[0, 0, 1, 1, 0, 1, 0, 0, 1, 0])
self._ref_bin_heights_manual = np.array(
[5, 4, 10, 3, 0, 15, 7, 4, 20, 1])
self._ref_bin_edges_manual_variablespacing = [
0, 2, 3, 3.1, 3.2, 3.3, 3.4, 7, 8.5, 9, 10
]
self._ref_data_manual_variablespacing = np.array(
[0, 1, 0, 0, 0, 1, 1, 0, 1, 0])
self._ref_data_auto = np.array([
0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1,
0, 0, 0
])
# test rebinning functionality
self._probe_bin_edges_variablespacing_withedges = [
0, 2, 3, 3.1, 3.2, 3.3, 3.4, 7, 8.5, 9, 10
] # OK
self._probe_bin_edges_variablespacing_noedges = [
2, 3, 3.1, 3.2, 3.3, 3.4, 7, 8.5, 9
] # OK
self._probe_bin_edges_variablespacing_wrongedges1 = [
0, 2, 3, 3.1, 3.2, 3.3, 3.4, 7, 8.5, 9, 12.3
] # fail
self._probe_bin_edges_variablespacing_wrongedges2 = [
-9, 2, 3, 3.1, 3.2, 3.3, 3.4, 7, 8.5, 9, 10
] # fail
self._probe_bin_edges_variablespacing_wrongedges3 = [
-3, 2, 3, 3.1, 3.2, 3.3, 3.4, 7, 8.5, 9, 22
] # fail
self._probe_bin_edges_variablespacing_wrongnumber = [
0, 2, 3, 3.1, 3.2, 3.3, 3.4, 7, 8.5, 10
] # fail
self._probe_bin_edges_variablespacing_unsorted = [
0, 2, 3, 8.5, 3.2, 3.3, 3.4, 7, 3.1, 9, 10
] # fail
self.hist_cont_binedges_auto = HistContainer(self._ref_n_bins_auto,
self._ref_n_bin_range,
bin_edges=None)
self.hist_cont_binedges_manual_equal = HistContainer(
self._ref_n_bins_manual,
self._ref_n_bin_range,
bin_edges=self._ref_bin_edges_manual_equalspacing)
self.hist_cont_binedges_manual_variable = HistContainer(
self._ref_n_bins_manual,
self._ref_n_bin_range,
bin_edges=self._ref_bin_edges_manual_variablespacing)
def test_property_size(self):
self.assertEqual(self.hist_cont_binedges_auto.size,
self._ref_n_bins_auto)
self.assertEqual(self.hist_cont_binedges_manual_equal.size,
self._ref_n_bins_manual)
self.assertEqual(self.hist_cont_binedges_manual_variable.size,
self._ref_n_bins_manual)
def test_property_low(self):
self.assertEqual(self.hist_cont_binedges_auto.low,
self._ref_n_bin_range[0])
self.assertEqual(self.hist_cont_binedges_manual_equal.low,
self._ref_n_bin_range[0])
self.assertEqual(self.hist_cont_binedges_manual_variable.low,
self._ref_n_bin_range[0])
def test_property_high(self):
self.assertEqual(self.hist_cont_binedges_auto.high,
self._ref_n_bin_range[1])
self.assertEqual(self.hist_cont_binedges_manual_equal.high,
self._ref_n_bin_range[1])
self.assertEqual(self.hist_cont_binedges_manual_variable.high,
self._ref_n_bin_range[1])
def test_fill_empty_binedges_auto_compare_data(self):
self.hist_cont_binedges_auto.fill(self._ref_entries)
self.assertTrue(
np.allclose(self.hist_cont_binedges_auto.data,
self._ref_data_auto))
def test_fill_empty_binedges_manual_equal_compare_data(self):
self.hist_cont_binedges_manual_equal.fill(self._ref_entries)
self.assertTrue(
np.allclose(self.hist_cont_binedges_manual_equal.data,
self._ref_data_manual_equalspacing))
def test_fill_empty_binedges_manual_variable_compare_data(self):
self.hist_cont_binedges_manual_variable.fill(self._ref_entries)
self.assertTrue(
np.allclose(self.hist_cont_binedges_manual_variable.data,
self._ref_data_manual_variablespacing))
def test_fill_empty_binedges_auto_rebin_manual_equal_compare_data(self):
self.hist_cont_binedges_auto.fill(self._ref_entries)
self.hist_cont_binedges_auto.rebin(
self._ref_bin_edges_manual_equalspacing)
self.assertTrue(
np.allclose(self.hist_cont_binedges_auto.data,
self._ref_data_manual_equalspacing))
def test_fill_empty_binedges_auto_rebin_manual_variable_compare_data(self):
self.hist_cont_binedges_auto.fill(self._ref_entries)
self.hist_cont_binedges_auto.rebin(
self._ref_bin_edges_manual_variablespacing)
self.assertTrue(
np.allclose(self.hist_cont_binedges_auto.data,
self._ref_data_manual_variablespacing))
def test_manual_bin_height(self):
self.hist_cont_binedges_manual_equal.set_bins(
self._ref_bin_heights_manual)
self.assertTrue(
np.alltrue(self.hist_cont_binedges_manual_equal.data ==
self._ref_bin_heights_manual))
self.assertTrue(self.hist_cont_binedges_manual_equal._manual_heights)
def test_construct_bin_edges_variablespacing_withedges(self):
_hc = HistContainer(
self._ref_n_bins_manual,
self._ref_n_bin_range,
bin_edges=self._probe_bin_edges_variablespacing_withedges)
def test_construct_bin_edges_variablespacing_noedges(self):
_hc = HistContainer(
self._ref_n_bins_manual,
self._ref_n_bin_range,
bin_edges=self._probe_bin_edges_variablespacing_noedges)
def test_raise_construct_bin_edges_variablespacing_wrongedges1(self):
with self.assertRaises(HistContainerException):
_hc = HistContainer(
self._ref_n_bins_manual,
self._ref_n_bin_range,
bin_edges=self._probe_bin_edges_variablespacing_wrongedges1)
def test_raise_construct_bin_edges_variablespacing_wrongedges2(self):
with self.assertRaises(HistContainerException):
_hc = HistContainer(
self._ref_n_bins_manual,
self._ref_n_bin_range,
bin_edges=self._probe_bin_edges_variablespacing_wrongedges2)
def test_raise_construct_bin_edges_variablespacing_wrongedges3(self):
with self.assertRaises(HistContainerException):
_hc = HistContainer(
self._ref_n_bins_manual,
self._ref_n_bin_range,
bin_edges=self._probe_bin_edges_variablespacing_wrongedges3)
def test_raise_construct_bin_edges_variablespacing_wrongnumber(self):
with self.assertRaises(HistContainerException):
_hc = HistContainer(
self._ref_n_bins_manual,
self._ref_n_bin_range,
bin_edges=self._probe_bin_edges_variablespacing_wrongnumber)
def test_raise_construct_bin_edges_variablespacing_unsorted(self):
with self.assertRaises(HistContainerException):
_hc = HistContainer(
self._ref_n_bins_manual,
self._ref_n_bin_range,
bin_edges=self._probe_bin_edges_variablespacing_unsorted)
def test_raise_add_same_error_name_twice(self):
self.hist_cont_binedges_auto.add_error(0.1,
name="MyNewError",
correlation=0,
relative=False)
with self.assertRaises(DataContainerException):
self.hist_cont_binedges_auto.add_error(0.1,
name="MyNewError",
correlation=0,
relative=False)
def test_raise_get_inexistent_error(self):
with self.assertRaises(DataContainerException):
self.hist_cont_binedges_auto.get_error("MyInexistentError")
def test_raise_manual_bin_heights_rebin(self):
self.hist_cont_binedges_manual_equal.set_bins(
self._ref_bin_heights_manual)
with self.assertRaises(DataContainerException):
self.hist_cont_binedges_manual_equal.rebin(
self._ref_bin_edges_manual_variablespacing)
class TestParameterConstraintInHistFit(unittest.TestCase):
@staticmethod
def _model_function(x, a, b):
return a * x + b
@staticmethod
def _model_function_antiderivative(x, a, b):
return 0.5 * a * x ** 2 + b * x
def _expected_profile_diff(self, res, cov_mat_inv):
return res.dot(cov_mat_inv).dot(res)
def _test_consistency(self, constrained_fit, par_cov_mat_inv):
constrained_fit.do_fit()
_cost_function = constrained_fit._fitter._fcn_wrapper
for _i in range(4):
for _j in range(9):
_a = self._test_par_values[_i, 0, _j]
_b = self._test_par_values[_i, 1, _j]
_profile_constrained = _cost_function(self._test_par_values[_i, 0, _j], self._test_par_values[_i, 1, _j])
_diff = _profile_constrained - self._profile_no_constraints[_i, _j]
_expected_profile_diff = self._expected_profile_diff(self._test_par_res[_i, _j], par_cov_mat_inv)
self.assertTrue(np.abs(_diff - _expected_profile_diff) < 1e-12)
def setUp(self):
_bin_edges = [0.0, 1.0, 2.0, 3.0, 4.0, 5.0]
_data = [
0.5,
1.5, 1.5,
2.5, 2.5, 2.5,
3.5, 3.5, 3.5, 3.5,
4.5, 4.5, 4.5, 4.5, 4.5
]
self._means = np.array([3.654, 7.789])
self._vars = np.array([2.467, 1.543])
self._cov_mat_uncor = np.array([[self._vars[0], 0.0], [0.0, self._vars[1]]])
self._cov_mat_uncor_inv = np.linalg.inv(self._cov_mat_uncor)
self._cov_mat_cor = np.array([[self._vars[0], 0.1], [0.1, self._vars[1]]])
self._cov_mat_cor_inv = np.linalg.inv(self._cov_mat_cor)
self._cov_mat_simple_a_inv = np.array([[1.0 / self._vars[0], 0.0], [0.0, 0.0]])
self._cov_mat_simple_b_inv = np.array([[0.0, 0.0], [0.0, 1.0 / self._vars[1]]])
self._data_container = HistContainer(n_bins=5, bin_range=(0.0, 5.0), fill_data=_data, dtype=float)
self._data_container.add_error(err_val=1.0)
_a_test = np.linspace(start=1, stop=2, num=9, endpoint=True)
_b_test = np.linspace(start=2, stop=3, num=9, endpoint=True)
self._test_par_values = np.zeros((4, 2, 9))
self._test_par_values[0, 0] = _a_test
self._test_par_values[1, 1] = _b_test
self._test_par_values[2, 0] = _a_test
self._test_par_values[2, 1] = _b_test
self._test_par_values[3, 0] = _a_test
self._test_par_values[3, 1] = _b_test[::-1] # reverse order
self._test_par_res = self._test_par_values - self._means.reshape((1, 2, 1))
self._test_par_res = np.transpose(self._test_par_res, axes=(0, 2, 1))
self._fit_no_constraints = HistFit(self._data_container, model_density_function=self._model_function,
model_density_antiderivative=self._model_function_antiderivative)
self._fit_no_constraints.do_fit()
_cost_function = self._fit_no_constraints._fitter._fcn_wrapper
self._profile_no_constraints = np.zeros((4, 9))
for _i in range(4):
for _j in range(9):
self._profile_no_constraints[_i, _j] = _cost_function(
self._test_par_values[_i, 0, _j],
self._test_par_values[_i, 1, _j])
def test_bad_input_exception(self):
_fit_with_constraint = HistFit(self._data_container, model_density_function=self._model_function,
model_density_antiderivative=self._model_function_antiderivative)
with self.assertRaises(HistFitException):
_fit_with_constraint.add_parameter_constraint('c', 1.0, 1.0)
with self.assertRaises(HistFitException):
_fit_with_constraint.add_matrix_parameter_constraint(['a', 'c'], [1.0, 2.0], [[0.2, 0.0], [0.0, 0.1]])
with self.assertRaises(HistFitException):
_fit_with_constraint.add_matrix_parameter_constraint(['a'], [1.0, 2.0], [[0.2, 0.0], [0.0, 0.1]])
def test_fit_profile_cov_mat_uncorrelated(self):
_fit_with_constraint = HistFit(self._data_container, model_density_function=self._model_function,
model_density_antiderivative=self._model_function_antiderivative)
_fit_with_constraint.add_matrix_parameter_constraint(['a', 'b'], self._means, self._cov_mat_uncor)
self._test_consistency(_fit_with_constraint, self._cov_mat_uncor_inv)
_fit_with_constraint_alt = HistFit(self._data_container, model_density_function=self._model_function,
model_density_antiderivative=self._model_function_antiderivative)
_fit_with_constraint_alt.add_parameter_constraint('a', self._means[0], np.sqrt(self._vars[0]))
_fit_with_constraint_alt.add_parameter_constraint('b', self._means[1], np.sqrt(self._vars[1]))
self._test_consistency(_fit_with_constraint_alt, self._cov_mat_uncor_inv)
def test_fit_profile_cov_mat_correlated(self):
_fit_with_constraint = HistFit(self._data_container, model_density_function=self._model_function,
model_density_antiderivative=self._model_function_antiderivative)
_fit_with_constraint.add_matrix_parameter_constraint(['a', 'b'], self._means, self._cov_mat_cor)
self._test_consistency(_fit_with_constraint, self._cov_mat_cor_inv)
def test_fit_profile_simple_a(self):
_fit_with_constraint = HistFit(self._data_container, model_density_function=self._model_function,
model_density_antiderivative=self._model_function_antiderivative)
_fit_with_constraint.add_parameter_constraint('a', self._means[0], np.sqrt(self._vars[0]))
self._test_consistency(_fit_with_constraint, self._cov_mat_simple_a_inv)
def test_fit_profile_simple_b(self):
_fit_with_constraint = HistFit(self._data_container, model_density_function=self._model_function,
model_density_antiderivative=self._model_function_antiderivative)
_fit_with_constraint.add_parameter_constraint('b', self._means[1], np.sqrt(self._vars[1]))
self._test_consistency(_fit_with_constraint, self._cov_mat_simple_b_inv)
class TestHistContainerYamlRepresentation(unittest.TestCase):
def setUp(self):
_data = [1, 1, 4, 1, 1, 4, 2, 5, 5, 5, 2, 4, 2, 2, 18, 2, 8, 8, 9]
_nbins = 3
self._container = HistContainer(n_bins=_nbins,
bin_range=(0.0, 10.),
fill_data=_data)
self._container.add_error(err_val=0.1,
name='SC5A_1',
correlation=0.5,
relative=False)
self._container.add_error(
err_val=[i / 10. for i in range(1, _nbins + 1)],
name='SC5A_2',
correlation=0.5,
relative=False)
self._container.add_matrix_error(err_matrix=np.eye(_nbins) * 0.1,
name='MCov',
relative=False,
matrix_type='covariance')
self._container.add_matrix_error(err_matrix=np.eye(_nbins),
name='MCor',
relative=False,
matrix_type='correlation',
err_val=0.03)
self._roundtrip_stringstream = IOStreamHandle(StringIO())
self._testfile_stringstream = IOStreamHandle(
StringIO(TEST_DATASET_HIST))
self._roundtrip_streamreader = DataContainerYamlReader(
self._roundtrip_stringstream)
self._roundtrip_streamwriter = DataContainerYamlWriter(
self._container, self._roundtrip_stringstream)
self._testfile_streamreader = DataContainerYamlReader(
self._testfile_stringstream)
self._testfile_stringstream_missing_keyword = IOStreamHandle(
StringIO(TEST_DATASET_HIST_MSSING_KEYWORD))
self._testfile_stringstream_extra_keyword = IOStreamHandle(
StringIO(TEST_DATASET_HIST_EXTRA_KEYWORD))
self._testfile_streamreader_missing_keyword = DataContainerYamlReader(
self._testfile_stringstream_missing_keyword)
self._testfile_streamreader_extra_keyword = DataContainerYamlReader(
self._testfile_stringstream_extra_keyword)
self._ref_testfile_n_bins = 2
self._ref_testfile_bin_range = [1.0, 3.0]
self._ref_testfile_bin_edges = [1.0, 2.0, 3.0]
self._ref_testfile_data = [1, 1]
self._ref_testfile_err = [0.89442719, 1]
self._ref_testfile_cov_mat = np.array([[0.8, 0.3], [0.3, 1.0]])
self._ref_testfile_error_names = {
'ErrorOne', 'ErrorTwo', 'ErrorThree', 'ErrorFour'
}
def test_write_to_roundtrip_stringstream(self):
self._roundtrip_streamwriter.write()
def test_read_from_testfile_stream(self):
_read_container = self._testfile_streamreader.read()
self.assertTrue(isinstance(_read_container, HistContainer))
self.assertTrue(_read_container.n_bins == self._ref_testfile_n_bins)
self.assertTrue(
np.allclose(_read_container.bin_range,
self._ref_testfile_bin_range))
self.assertTrue(
np.allclose(_read_container.bin_edges,
self._ref_testfile_bin_edges))
self.assertTrue(
np.allclose(_read_container.data, self._ref_testfile_data))
self.assertTrue(
np.allclose(_read_container.err, self._ref_testfile_err))
self.assertTrue(
np.allclose(
_read_container.cov_mat,
self._ref_testfile_cov_mat,
))
# check that the error names are the same
self.assertEqual(set(_read_container._error_dicts.keys()),
self._ref_testfile_error_names)
def test_read_from_testfile_stream_missing_keyword(self):
with self.assertRaises(YamlReaderException):
self._testfile_streamreader_missing_keyword.read()
def test_read_from_testfile_stream_extra_keyword(self):
with self.assertRaises(YamlReaderException):
self._testfile_streamreader_extra_keyword.read()
def test_round_trip_with_stringstream(self):
self._roundtrip_streamwriter.write()
self._roundtrip_stringstream.seek(0) # return to beginning
_read_container = self._roundtrip_streamreader.read()
self.assertTrue(isinstance(_read_container, HistContainer))
self.assertTrue(self._container.n_bins == _read_container.n_bins)
self.assertTrue(
np.allclose(self._container.bin_range, _read_container.bin_range))
self.assertTrue(
np.allclose(self._container.bin_edges, _read_container.bin_edges))
# compare data members
self.assertTrue(np.allclose(self._container.data,
_read_container.data))
# compare (total) errors and cov mats
# TODO: compare individual error components -> nontrivial because error ids change...
self.assertTrue(np.allclose(self._container.err, _read_container.err))
self.assertTrue(
np.allclose(self._container.cov_mat, _read_container.cov_mat))
# check that the error names are the same
self.assertEqual(set(self._container._error_dicts.keys()),
set(_read_container._error_dicts.keys()))
