def test_combine_heuristics(heuristic1, heuristic2, weights):
np.random.seed(1337)
predictions = [distributions_3d, distributions_5d]
with pytest.raises(ValueError) as excinfo:
heuristic1(predictions)
assert "CombineHeuristics" in str(excinfo.value)
if isinstance(heuristic1,
Certainty) and not isinstance(heuristic2, Certainty):
with pytest.raises(Exception) as e_info:
heuristics = CombineHeuristics([heuristic1, heuristic2],
weights=weights,
reduction='mean')
assert 'heuristics should have the same value for `revesed` parameter' in str(
e_info.value)
else:
heuristics = CombineHeuristics([heuristic1, heuristic2],
weights=weights,
reduction='mean')
if isinstance(heuristic1, Certainty) and isinstance(
heuristic2, Certainty):
assert not heuristics.reversed
else:
assert heuristics.reversed
ranks = heuristics(predictions)
assert np.all(ranks == [1, 2, 0]
), "Combine Heuristics is not right {}".format(ranks)
def test_combine_heuristics_reorder_list():
# we are just testing if given calculated uncertainty measures for chunks of data
# the `reorder_indices` would make correct decision. Here index 0 has the
# highest uncertainty chosen but both methods (uncertainties1 and uncertainties2)
bald_firstchunk = np.array([0.98])
bald_secondchunk = np.array([0.87, 0.68])
variance_firstchunk = np.array([0.76])
variance_secondchunk = np.array([0.63, 0.48])
streaming_prediction = [[bald_firstchunk, variance_firstchunk],
[bald_secondchunk, variance_secondchunk]]
heuristics = CombineHeuristics([BALD(), Variance()], weights=[0.5, 0.5],
reduction='mean')
ranks = heuristics.reorder_indices(streaming_prediction)
assert np.all(ranks == [0, 1, 2]), "Combine Heuristics is not right {}".format(ranks)
def test_combine_heuristics_uncertainty_generator():
np.random.seed(1337)
prediction_chunks = [chunks(distributions_3d, 2), chunks(distributions_5d, 2)]
predictions = [distributions_3d, distributions_5d]
heuristics = CombineHeuristics([BALD(), Variance()], weights=[0.5, 0.5],
reduction='mean')
assert np.allclose(
heuristics.get_uncertainties(predictions),
heuristics.get_uncertainties(prediction_chunks),
)
prediction_chunks = [chunks(distributions_3d, 2), chunks(distributions_5d, 2)]
ranks = heuristics(prediction_chunks)
assert np.all(ranks == [1, 2, 0]), "Combine Heuristics is not right {}".format(ranks)