def test_combine_heuristics(heuristic1, heuristic2, weights): np.random.seed(1337) predictions = [distributions_3d, distributions_5d] with pytest.raises(ValueError) as excinfo: heuristic1(predictions) assert "CombineHeuristics" in str(excinfo.value) if isinstance(heuristic1, Certainty) and not isinstance(heuristic2, Certainty): with pytest.raises(Exception) as e_info: heuristics = CombineHeuristics([heuristic1, heuristic2], weights=weights, reduction='mean') assert 'heuristics should have the same value for `revesed` parameter' in str( e_info.value) else: heuristics = CombineHeuristics([heuristic1, heuristic2], weights=weights, reduction='mean') if isinstance(heuristic1, Certainty) and isinstance( heuristic2, Certainty): assert not heuristics.reversed else: assert heuristics.reversed ranks = heuristics(predictions) assert np.all(ranks == [1, 2, 0] ), "Combine Heuristics is not right {}".format(ranks)
def test_combine_heuristics_reorder_list(): # we are just testing if given calculated uncertainty measures for chunks of data # the `reorder_indices` would make correct decision. Here index 0 has the # highest uncertainty chosen but both methods (uncertainties1 and uncertainties2) bald_firstchunk = np.array([0.98]) bald_secondchunk = np.array([0.87, 0.68]) variance_firstchunk = np.array([0.76]) variance_secondchunk = np.array([0.63, 0.48]) streaming_prediction = [[bald_firstchunk, variance_firstchunk], [bald_secondchunk, variance_secondchunk]] heuristics = CombineHeuristics([BALD(), Variance()], weights=[0.5, 0.5], reduction='mean') ranks = heuristics.reorder_indices(streaming_prediction) assert np.all(ranks == [0, 1, 2]), "Combine Heuristics is not right {}".format(ranks)
def test_combine_heuristics_uncertainty_generator(): np.random.seed(1337) prediction_chunks = [chunks(distributions_3d, 2), chunks(distributions_5d, 2)] predictions = [distributions_3d, distributions_5d] heuristics = CombineHeuristics([BALD(), Variance()], weights=[0.5, 0.5], reduction='mean') assert np.allclose( heuristics.get_uncertainties(predictions), heuristics.get_uncertainties(prediction_chunks), ) prediction_chunks = [chunks(distributions_3d, 2), chunks(distributions_5d, 2)] ranks = heuristics(prediction_chunks) assert np.all(ranks == [1, 2, 0]), "Combine Heuristics is not right {}".format(ranks)