def test_compute(self): multiclass_paired_metrics = metrics.MulticlassPairedMetricsImpl() indices = ['7f7f85', '345ac4', '3a3112', '88bcda'] metas = [{'parentId': '345ac4'}, {}, {}, {'parentId': '3a3112'}] # No swaps. result = multiclass_paired_metrics.compute_with_metadata( ['1', '1', '0', '0'], [[0, 1], [0, 1], [1, 0], [1, 0]], types.CategoryLabel(), types.MulticlassPreds(vocab=['0', '1'], null_idx=0), indices, metas) testing_utils.assert_deep_almost_equal(self, result, { 'mean_jsd': 0.0, 'num_pairs': 2, 'swap_rate': 0.0 }) # One swap. result = multiclass_paired_metrics.compute_with_metadata( ['1', '1', '0', '0'], [[0, 1], [1, 0], [1, 0], [1, 0]], types.CategoryLabel(), types.MulticlassPreds(vocab=['0', '1'], null_idx=0), indices, metas) testing_utils.assert_deep_almost_equal(self, result, { 'mean_jsd': 0.34657, 'num_pairs': 2, 'swap_rate': 0.5 }) # Two swaps. result = multiclass_paired_metrics.compute_with_metadata( ['1', '1', '0', '0'], [[0, 1], [1, 0], [1, 0], [0, 1]], types.CategoryLabel(), types.MulticlassPreds(vocab=['0', '1'], null_idx=0), indices, metas) testing_utils.assert_deep_almost_equal(self, result, { 'mean_jsd': 0.69315, 'num_pairs': 2, 'swap_rate': 1.0 }) # Two swaps, no null index. result = multiclass_paired_metrics.compute_with_metadata( ['1', '1', '0', '0'], [[0, 1], [1, 0], [1, 0], [0, 1]], types.CategoryLabel(), types.MulticlassPreds(vocab=['0', '1']), indices, metas) testing_utils.assert_deep_almost_equal(self, result, { 'mean_jsd': 0.69315, 'num_pairs': 2, 'swap_rate': 1.0 }) # Empty predictions, indices, and meta. result = multiclass_paired_metrics.compute_with_metadata( [], [], types.CategoryLabel(), types.MulticlassPreds(vocab=['0', '1'], null_idx=0), [], []) testing_utils.assert_deep_almost_equal(self, result, {})
def test_classification_cat(self): config = { 'feature': 'cats', } result = self.pdp.run_with_metadata([self.dataset.indexed_examples[0]], self.class_model, self.dataset, config=config) expected = {'One': [0.49, 0.51], 'None': [0.99, 0.01]} testing_utils.assert_deep_almost_equal(self, result['probas'], expected)
def test_regression_cat(self): config = { 'feature': 'cats', } result = self.pdp.run_with_metadata([self.dataset.indexed_examples[0]], self.reg_model, self.dataset, config=config) expected = {'One': 2.0, 'None': 1.0} testing_utils.assert_deep_almost_equal(self, result['score'], expected)
def test_regression_num(self): config = { 'feature': 'num', } result = self.pdp.run_with_metadata([self.dataset.indexed_examples[0]], self.reg_model, self.dataset, config=config) expected = {1.0: 2.0, 2.0: 3.0, 3.0: 4.0, 4.0: 5.0, 5.0: 6.0, 6.0: 7.0, 7.0: 8.0, 8.0: 9.0, 9.0: 10.0, 10.0: 11.0} testing_utils.assert_deep_almost_equal(self, result['score'], expected)
def test_multiple_inputs(self): config = { 'feature': 'num', } result = self.pdp.run_with_metadata(self.dataset.indexed_examples[0:2], self.reg_model, self.dataset, config=config) expected = {1.0: 1.5, 2.0: 2.5, 3.0: 3.5, 4.0: 4.5, 5.0: 5.5, 6.0: 6.5, 7.0: 7.5, 8.0: 8.5, 9.0: 9.5, 10.0: 10.5} testing_utils.assert_deep_almost_equal(self, result['score'], expected)
def test_tcav_sample_from_positive(self): # Tests the case where more concept examples are passed than non-concept # examples, so the concept set is sampled from the concept examples. random.seed(0) # Sets seed since create_comparison_splits() uses random. # Basic test with dummy outputs from the model. examples = [ {'sentence': 'a'}, {'sentence': 'b'}, {'sentence': 'c'}, {'sentence': 'd'}, {'sentence': 'e'}, {'sentence': 'f'}, {'sentence': 'g'}, {'sentence': 'h'}] indexed_inputs = [{'id': caching.input_hash(ex), 'data': ex} for ex in examples] dataset = lit_dataset.IndexedDataset(id_fn=caching.input_hash, indexed_examples=indexed_inputs) config = { 'concept_set_ids': [indexed_inputs[0]['id'], indexed_inputs[2]['id'], indexed_inputs[3]['id'], indexed_inputs[4]['id'], indexed_inputs[7]['id']], 'class_to_explain': '1', 'grad_layer': 'cls_grad', 'random_state': 0 } result = self.tcav.run_with_metadata(indexed_inputs, self.model, dataset, config=config) self.assertLen(result, 1) expected = { 'p_val': 0.80489, 'random_mean': 0.53333, 'result': { 'score': 0.8, 'cos_sim': [ 0.09527, -0.20442, 0.05141, 0.14985, 0.06750, -0.28244, -0.11022, -0.14479 ], 'dot_prods': [ 152.48776, -335.64998, 82.99588, 247.80113, 109.53684, -461.81805, -181.29095, -239.47817 ], 'accuracy': 1.0 } } testing_utils.assert_deep_almost_equal(self, expected, result[0])
def test_compute(self): multiclass_metrics = metrics.MulticlassMetricsImpl() # All correct predictions. result = multiclass_metrics.compute( ['1', '2', '0', '1'], [[0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 1, 0]], types.CategoryLabel(), types.MulticlassPreds(vocab=['0', '1', '2'], null_idx=0)) testing_utils.assert_deep_almost_equal(self, result, { 'accuracy': 1.0, 'f1': 1.0, 'precision': 1.0, 'recall': 1.0 }) # Some incorrect predictions. result = multiclass_metrics.compute( ['1', '2', '0', '1'], [[.1, .4, .5], [0, .1, .9], [.1, 0, .9], [0, 1, 0]], types.CategoryLabel(), types.MulticlassPreds(vocab=['0', '1', '2'], null_idx=0)) testing_utils.assert_deep_almost_equal(self, result, { 'accuracy': 0.5, 'f1': 0.57143, 'precision': 0.5, 'recall': 0.66667 }) # All incorrect predictions. result = multiclass_metrics.compute( ['1', '2', '0', '1'], [[.1, .4, .5], [.2, .7, .1], [.1, 0, .9], [1, 0, 0]], types.CategoryLabel(), types.MulticlassPreds(vocab=['0', '1', '2'], null_idx=0)) testing_utils.assert_deep_almost_equal(self, result, { 'accuracy': 0.0, 'f1': 0.0, 'precision': 0.0, 'recall': 0.0 }) # No null index. result = multiclass_metrics.compute( ['1', '2', '0', '1'], [[.1, .4, .5], [0, .1, .9], [.1, 0, .9], [0, 1, 0]], types.CategoryLabel(), types.MulticlassPreds(vocab=['0', '1', '2'])) testing_utils.assert_deep_almost_equal(self, result, {'accuracy': 0.5}) # Empty labels and predictions result = multiclass_metrics.compute([], [], types.CategoryLabel(), types.MulticlassPreds( vocab=['0', '1', '2'], null_idx=0)) testing_utils.assert_deep_almost_equal(self, result, {})
def test_tcav(self): random.seed(0) # Sets seed since create_comparison_splits() uses random. # Basic test with dummy outputs from the model. examples = [ {'sentence': 'a'}, {'sentence': 'b'}, {'sentence': 'c'}, {'sentence': 'd'}, {'sentence': 'e'}, {'sentence': 'f'}, {'sentence': 'g'}, {'sentence': 'h'}, {'sentence': 'i'}] indexed_inputs = [{'id': caching.input_hash(ex), 'data': ex} for ex in examples] dataset = lit_dataset.IndexedDataset(id_fn=caching.input_hash, indexed_examples=indexed_inputs) config = { 'concept_set_ids': [indexed_inputs[0]['id'], indexed_inputs[2]['id'], indexed_inputs[3]['id'], indexed_inputs[7]['id']], 'class_to_explain': '1', 'grad_layer': 'cls_grad', 'random_state': 0, 'dataset_name': 'test' } result = self.tcav.run_with_metadata(indexed_inputs, self.model, dataset, config=config) self.assertLen(result, 1) expected = { 'p_val': 0.13311, 'random_mean': 0.56667, 'result': { 'score': 0.33333, 'cos_sim': [ 0.088691, -0.12179, 0.16013, 0.24840, -0.09793, 0.05166, -0.21578, -0.06560, -0.14759 ], 'dot_prods': [ 189.085096, -266.36317, 344.350498, 547.144949, -211.663965, 112.502439, -472.72066, -144.529598, -323.31888 ], 'accuracy': 0.66667 } } testing_utils.assert_deep_almost_equal(self, expected, result[0])
def test_class_num(self): config = { 'feature': 'num', } result = self.pdp.run_with_metadata([self.dataset.indexed_examples[0]], self.class_model, self.dataset, config=config) expected = {1.0: [0.49, 0.51], 2.0: [0.48, 0.52], 3.0: [0.47, 0.53], 4.0: [0.46, 0.54], 5.0: [0.45, 0.55], 6.0: [0.44, 0.56], 7.0: [0.43, 0.57], 8.0: [0.42, 0.58], 9.0: [0.41, 0.59], 10.0: [0.4, 0.6]} testing_utils.assert_deep_almost_equal(self, result['probas'], expected)
def test_compute(self): corpusblue_metrics = metrics.CorpusBLEU() # All correct predictions. result = corpusblue_metrics.compute( ['This is a test.', 'Test two', 'A third test example'], ['This is a test.', 'Test two', 'A third test example'], types.GeneratedText(), types.GeneratedText()) testing_utils.assert_deep_almost_equal(self, result, {'corpus_bleu': 100.00000}) # Some incorrect predictions. result = corpusblue_metrics.compute( ['This is a test.', 'Test one', 'A third test'], ['This is a test.', 'Test two', 'A third test example'], types.GeneratedText(), types.GeneratedText()) testing_utils.assert_deep_almost_equal(self, result, {'corpus_bleu': 68.037493}) result = corpusblue_metrics.compute( ['This is a test.', 'Test one', 'A third test'], ['these test.', 'Test two', 'A third test example'], types.GeneratedText(), types.GeneratedText()) testing_utils.assert_deep_almost_equal( self, result, {'corpus_bleu': 29.508062388758525}) # Empty labels and predictions result = corpusblue_metrics.compute([], [], types.GeneratedText(), types.GeneratedText()) testing_utils.assert_deep_almost_equal(self, result, {})
def test_run_nn(self): examples = [ { 'segment': 'a' }, { 'segment': 'b' }, { 'segment': 'c' }, ] indexed_inputs = [{ 'id': caching.input_hash(ex), 'data': ex } for ex in examples] model = TestModelNearestNeighbors() dataset = lit_dataset.IndexedDataset(id_fn=caching.input_hash, indexed_examples=indexed_inputs) config = { 'embedding_name': 'input_embs', 'num_neighbors': 2, } result = self.nearest_neighbors.run_with_metadata([indexed_inputs[1]], model, dataset, config=config) expected = { 'nearest_neighbors': [{ 'id': '1', 'nn_distance': 0.0 }, { 'id': '0', 'nn_distance': 1.7320508075688772 }] } self.assertLen(result, 1) testing_utils.assert_deep_almost_equal(self, expected, result[0])
def test_compute(self): regression_metrics = metrics.RegressionMetrics() # All correct predictions. result = regression_metrics.compute([1, 2, 3, 4], [1, 2, 3, 4], types.RegressionScore(), types.RegressionScore()) testing_utils.assert_deep_almost_equal(self, result, { 'mse': 0, 'pearsonr': 1.0, 'spearmanr': 1.0 }) # Some incorrect predictions. result = regression_metrics.compute([1, 2, 3, 4], [1, 2, 5.5, 6.3], types.RegressionScore(), types.RegressionScore()) testing_utils.assert_deep_almost_equal(self, result, { 'mse': 2.885, 'pearsonr': 0.96566, 'spearmanr': 1.0 }) # All incorrect predictions (and not monotonic). result = regression_metrics.compute([1, 2, 3, 4], [-5, -10, 5, 6], types.RegressionScore(), types.RegressionScore()) testing_utils.assert_deep_almost_equal(self, result, { 'mse': 47.0, 'pearsonr': 0.79559, 'spearmanr': 0.799999 }) # Empty labels and predictions result = regression_metrics.compute([], [], types.RegressionScore(), types.RegressionScore()) testing_utils.assert_deep_almost_equal(self, result, {})
def test_relative_tcav(self): # Tests passing in a negative set. random.seed(0) # Sets seed since create_comparison_splits() uses random. # Basic test with dummy outputs from the model. examples = [ {'sentence': 'happy'}, # 0 {'sentence': 'sad'}, # 1 {'sentence': 'good'}, # 2 {'sentence': 'bad'}, # 3 {'sentence': 'pretty'}, # 4 {'sentence': 'ugly'}, # 5 {'sentence': 'sweet'}, # 6 {'sentence': 'bitter'}, # 7 {'sentence': 'well'}, # 8 {'sentence': 'poor'}, # 9 {'sentence': 'compelling'}, # 10 {'sentence': 'boring'}, # 11 {'sentence': 'pleasing'}, # 12 {'sentence': 'gross'}, # 13 {'sentence': 'blue'}, # 14 {'sentence': 'red'}, # 15 {'sentence': 'flower'}, # 16 {'sentence': 'bee'}, # 17 {'sentence': 'snake'}, # 18 {'sentence': 'windshield'}, # 19 {'sentence': 'plant'}, # 20 {'sentence': 'scary'}, # 21 {'sentence': 'pencil'}, # 22 {'sentence': 'hello'} # 23 ] indexed_inputs = [{'id': caching.input_hash(ex), 'data': ex} for ex in examples] dataset = lit_dataset.IndexedDataset(id_fn=caching.input_hash, indexed_examples=indexed_inputs) # This first example doesn't have enough examples for statistical testing, # so the returned p-value is None. config = { 'concept_set_ids': [indexed_inputs[0]['id'], indexed_inputs[2]['id'], indexed_inputs[4]['id']], 'negative_set_ids': [indexed_inputs[1]['id'], indexed_inputs[3]['id'], indexed_inputs[5]['id']], 'class_to_explain': '1', 'grad_layer': 'cls_grad', 'random_state': 0 } result = self.tcav.run_with_metadata(indexed_inputs, self.model, dataset, config=config) self.assertLen(result, 1) expected = { 'result': { 'score': 1.0, 'cos_sim': [ 0.9999999581246426, 0.049332143689572144, 0.8987945047547466, -0.41858423757857954, 0.6908297036543664, -0.5167857909664919, 0.8423017503220364, -0.005793079244916016, 0.8334491603894322, -0.4054645113448612, 0.7616102123736647, -0.4578596155267783, 0.8366905563807711, -0.27390786544756535, 0.7325538474066896, 0.5190287630768531, 0.8145227936096425, 0.02005592868363552, -0.1143256029298114, -0.1221480700842533, 0.6852995739227957, 0.3984620730733816, 0.5211149530112407, 0.5909723902471223 ], 'dot_prods': [ 1385.1480610241554, 69.95638452724207, 1239.4947646060161, -595.253135700978, 971.5880156862692, -725.0749813217176, 1182.8641913758102, -8.149647641120662, 1146.5803071544124, -576.4043054391316, 1038.3510704649307, -648.097269442522, 1154.4720122394317, -378.32103870822493, 1024.066390571124, 738.6959135414066, 1139.7963358416857, 28.691395032352318, -167.37808507284706, -176.4474746971391, 959.5159619261449, 562.8772536987927, 716.7270332848395, 840.7031847912738 ], 'accuracy': 0.5 }, 'p_val': None, 'random_mean': 0.9285714285714286, 'split_size': 3, 'num_runs': 1 } testing_utils.assert_deep_almost_equal(self, expected, result[0]) # This example has enough inputs for two runs of size 3. config = { 'concept_set_ids': [ indexed_inputs[1]['id'], indexed_inputs[2]['id'], indexed_inputs[4]['id'], indexed_inputs[5]['id'], indexed_inputs[10]['id'], indexed_inputs[9]['id'] ], 'negative_set_ids': [ indexed_inputs[0]['id'], indexed_inputs[3]['id'], indexed_inputs[12]['id'], indexed_inputs[6]['id'], indexed_inputs[7]['id'], indexed_inputs[8]['id'] ], 'class_to_explain': '0', 'grad_layer': 'cls_grad', 'random_state': 0 } result = self.tcav.run_with_metadata(indexed_inputs, self.model, dataset, config=config) self.assertLen(result, 1) expected = { 'result': { 'score': 0.0, 'cos_sim': [ 0.2731987606830683, 0.427838045403812, 0.3166440584420665, -0.1358964965831398, 0.5616614702946262, -0.16511808390168164, -0.05103355252438478, -0.16945565920473257, 0.28148962348967155, -0.18169036476392003, 0.33244873698665106, -0.13316476546155087, 0.15226772288202886, -0.05534469666649352, 0.2886150002073456, 0.33888135113008555, 0.12875301375254147, 0.046908665182593096, -0.052445114502024985, 0.088858405172313, 0.219517174438115, 0.35833013079793435, 0.2291162415605806, 0.3635686086637199 ], 'dot_prods': [ 452.17220644153525, 724.9460578876271, 521.776546745851, -230.9170522777958, 943.8754747127095, -276.8190148523963, -85.63511897570154, -284.8487792023684, 462.71830216201926, -308.62790255581496, 541.5830529968077, -225.2299308998058, 251.04716264718752, -91.33998249705493, 482.0991668852444, 576.3029773313335, 215.28329927312336, 80.18458502795752, -91.74640483442752, 153.37559992294862, 367.2562273288043, 604.8378479001944, 376.53473821563625, 618.003311205616 ], 'accuracy': 0.5 }, 'p_val': 0.42264973081037427, 'random_mean': 0.0, 'split_size': 3, 'num_runs': 2 } testing_utils.assert_deep_almost_equal(self, expected, result[0]) # This example has enough examples for three runs of size 3 and two runs of # size 5, and returns results with p-value < 0.05. config = { 'concept_set_ids': [indexed_inputs[0]['id'], indexed_inputs[1]['id'], indexed_inputs[2]['id'], indexed_inputs[3]['id'], indexed_inputs[4]['id'], indexed_inputs[5]['id'], indexed_inputs[6]['id'], indexed_inputs[7]['id'], indexed_inputs[8]['id'], indexed_inputs[9]['id']], 'negative_set_ids': [indexed_inputs[10]['id'], indexed_inputs[11]['id'], indexed_inputs[12]['id'], indexed_inputs[13]['id'], indexed_inputs[14]['id'], indexed_inputs[15]['id'], indexed_inputs[16]['id'], indexed_inputs[17]['id'], indexed_inputs[18]['id'], indexed_inputs[19]['id']], 'class_to_explain': '1', 'grad_layer': 'cls_grad', 'random_state': 0 } result = self.tcav.run_with_metadata(indexed_inputs, self.model, dataset, config=config) self.assertLen(result, 1) expected = [{ 'result': { 'score': 0.42857142857142855, 'cos_sim': [ -0.1107393877916321, -0.0993967046974328, -0.2214985917242054, 0.08132588965575606, -0.3590211572508748, 0.18708109817461333, 0.000724498781128839, 0.09700473783330398, -0.25015742815240055, 0.16108236033785076, -0.10283274286140846, 0.0972663321478731, -0.05924679176256152, -0.048499696342091746, -0.4357117016074766, -0.593245752003111, -0.3645147796989344, -0.5507605083253673, -0.27914997949782694, -0.30908550968594417, -0.5584676299422896, -0.16983339994284577, -0.42587740852240746, -0.37482298817032594 ], 'dot_prods': [ -261.4389298435066, -240.23776409902007, -520.6275907607769, 197.11495117497446, -860.6035066083074, 447.3775519523981, 1.7341104803878409, 232.59170976304426, -586.5576327736542, 390.2961568516803, -238.95427152619726, 234.6617547723058, -139.3334215524385, -114.17392512371171, -1038.149036709951, -1439.0663895591745, -869.3828698612926, -1342.899780229334, -696.569760699206, -760.9907977738051, -1332.7284530349625, -408.90435403478875, -998.3360993150825, -908.8111404537224 ], 'accuracy': 0.75 }, 'p_val': 0.04400624968940752, 'random_mean': 0.9642857142857143, 'split_size': 5, 'num_runs': 2 }] testing_utils.assert_deep_almost_equal(self, expected, result)