def output_spec(self) -> Spec: ret = {"tokens": lit_types.Tokens()} ret["tokens_" + self.config.text_a_name] = lit_types.Tokens() if self.config.text_b_name: ret["tokens_" + self.config.text_b_name] = lit_types.Tokens() if self.is_regression: ret["score"] = lit_types.RegressionScore( parent=self.config.label_name) else: ret["probas"] = lit_types.MulticlassPreds( parent=self.config.label_name, vocab=self.config.labels, null_idx=self.config.null_label_idx) ret["cls_emb"] = lit_types.Embeddings() # Gradients, if requested. if self.config.compute_grads: ret["token_grad_" + self.config.text_a_name] = lit_types.TokenGradients( align="tokens_" + self.config.text_a_name) if self.config.text_b_name: ret["token_grad_" + self.config.text_b_name] = lit_types.TokenGradients( align="tokens_" + self.config.text_b_name) # Attention heads, one field for each layer. for i in range(self.model.config.num_hidden_layers): ret[f"layer_{i}/attention"] = lit_types.AttentionHeads( align=("tokens", "tokens")) return ret
def output_spec(self) -> lit_types.Spec: return { "tokens": lit_types.Tokens(), "logits": lit_types.RegressionScore(), "cls_emb": lit_types.Embeddings(), "token_grad_sentence": lit_types.TokenGradients(align="tokens") }
def test_find_spec_keys(self): spec = { "score": types.RegressionScore(), "scalar_foo": types.Scalar(), "text": types.TextSegment(), "emb_0": types.Embeddings(), "emb_1": types.Embeddings(), "tokens": types.Tokens(), "generated_text": types.GeneratedText(), } self.assertEqual(["score"], utils.find_spec_keys(spec, types.RegressionScore)) self.assertEqual(["text", "tokens", "generated_text"], utils.find_spec_keys(spec, (types.TextSegment, types.Tokens))) self.assertEqual(["emb_0", "emb_1"], utils.find_spec_keys(spec, types.Embeddings)) self.assertEqual([], utils.find_spec_keys(spec, types.AttentionHeads)) # Check subclasses self.assertEqual( list(spec.keys()), utils.find_spec_keys(spec, types.LitType)) self.assertEqual(["text", "generated_text"], utils.find_spec_keys(spec, types.TextSegment)) self.assertEqual(["score", "scalar_foo"], utils.find_spec_keys(spec, types.Scalar))
def input_spec(self) -> Spec: ret = {} ret[self.config.text_a_name] = lit_types.TextSegment() if self.config.text_b_name: ret[self.config.text_b_name] = lit_types.TextSegment() if self.is_regression: ret[self.config.label_name] = lit_types.RegressionScore(required=False) else: ret[self.config.label_name] = lit_types.CategoryLabel( required=False, vocab=self.config.labels) return ret
def test_is_compatible(self): corpusblue_metrics = metrics.CorpusBLEU() # Only compatible with GeneratedText spec. self.assertTrue(corpusblue_metrics.is_compatible( types.GeneratedText())) self.assertFalse( corpusblue_metrics.is_compatible( types.MulticlassPreds(vocab=['']))) self.assertFalse( corpusblue_metrics.is_compatible(types.RegressionScore()))
def test_is_compatible(self): multiclass_metrics = metrics.MulticlassMetrics() # Only compatible with MulticlassPreds spec. self.assertTrue( multiclass_metrics.is_compatible( types.MulticlassPreds(vocab=['']))) self.assertFalse( multiclass_metrics.is_compatible(types.RegressionScore())) self.assertFalse( multiclass_metrics.is_compatible(types.GeneratedText()))
def set_up_pandas_dataframe_and_columns(): dataframe = pd.DataFrame.from_dict({ "feature_1": [1.0], "feature_2": [3.0], "label": [1.0] }) columns = collections.OrderedDict([ ("feature_1", lit_types.Scalar()), ("feature_2", lit_types.Scalar()), ("label", lit_types.RegressionScore()), ]) yield dataframe, columns
def output_spec(self) -> Spec: ret = {"tokens": lit_types.Tokens()} ret["tokens_" + self.config.text_a_name] = lit_types.Tokens( parent=self.config.text_a_name) if self.config.text_b_name: ret["tokens_" + self.config.text_b_name] = lit_types.Tokens( parent=self.config.text_b_name) if self.is_regression: ret["score"] = lit_types.RegressionScore(parent=self.config.label_name) else: ret["probas"] = lit_types.MulticlassPreds( parent=self.config.label_name, vocab=self.config.labels, null_idx=self.config.null_label_idx) ret["cls_emb"] = lit_types.Embeddings() # Average embeddings, one per layer including embeddings. for i in range(1 + self.model.config.num_hidden_layers): ret[f"layer_{i}/avg_emb"] = lit_types.Embeddings() ret["cls_grad"] = lit_types.Gradients( grad_for="cls_emb", grad_target_field_key="grad_class") # The input_embs_ and grad_class fields are used for Integrated Gradients. ret["input_embs_" + self.config.text_a_name] = lit_types.TokenEmbeddings( align="tokens_" + self.config.text_a_name) if self.config.text_b_name: ret["input_embs_" + self.config.text_b_name] = lit_types.TokenEmbeddings( align="tokens_" + self.config.text_b_name) # Gradients, if requested. if self.config.compute_grads: ret["grad_class"] = lit_types.CategoryLabel(required=False, vocab=self.config.labels) ret["token_grad_" + self.config.text_a_name] = lit_types.TokenGradients( align="tokens_" + self.config.text_a_name, grad_for="input_embs_" + self.config.text_a_name, grad_target_field_key="grad_class") if self.config.text_b_name: ret["token_grad_" + self.config.text_b_name] = lit_types.TokenGradients( align="tokens_" + self.config.text_b_name, grad_for="input_embs_" + self.config.text_b_name, grad_target_field_key="grad_class") # Attention heads, one field for each layer. for i in range(self.model.config.num_hidden_layers): ret[f"layer_{i+1}/attention"] = lit_types.AttentionHeads( align_in="tokens", align_out="tokens") return ret
def test_compute(self): regression_metrics = metrics.RegressionMetrics() # All correct predictions. result = regression_metrics.compute([1, 2, 3, 4], [1, 2, 3, 4], types.RegressionScore(), types.RegressionScore()) self.assertAlmostEqual(result, { 'mse': 0, 'pearsonr': 1.0, 'spearmanr': 1.0 }) # Some incorrect predictions. result = regression_metrics.compute([1, 2, 3, 4], [1, 2, 5.5, 6.3], types.RegressionScore(), types.RegressionScore()) self.assertAlmostEqual(result, { 'mse': 2.885, 'pearsonr': 0.9656642444980974, 'spearmanr': 1.0 }) # All incorrect predictions (and not monotonic). result = regression_metrics.compute([1, 2, 3, 4], [-5, -10, 5, 6], types.RegressionScore(), types.RegressionScore()) self.assertAlmostEqual( result, { 'mse': 47.0, 'pearsonr': 0.795592252958155, 'spearmanr': 0.7999999999999999 }) # Empty labels and predictions result = regression_metrics.compute([], [], types.RegressionScore(), types.RegressionScore()) self.assertAlmostEqual(result, {})
def test_compute(self): regression_metrics = metrics.RegressionMetrics() # All correct predictions. result = regression_metrics.compute([1, 2, 3, 4], [1, 2, 3, 4], types.RegressionScore(), types.RegressionScore()) testing_utils.assert_dicts_almost_equal(self, result, { 'mse': 0, 'pearsonr': 1.0, 'spearmanr': 1.0 }) # Some incorrect predictions. result = regression_metrics.compute([1, 2, 3, 4], [1, 2, 5.5, 6.3], types.RegressionScore(), types.RegressionScore()) testing_utils.assert_dicts_almost_equal(self, result, { 'mse': 2.885, 'pearsonr': 0.96566, 'spearmanr': 1.0 }) # All incorrect predictions (and not monotonic). result = regression_metrics.compute([1, 2, 3, 4], [-5, -10, 5, 6], types.RegressionScore(), types.RegressionScore()) testing_utils.assert_dicts_almost_equal(self, result, { 'mse': 47.0, 'pearsonr': 0.79559, 'spearmanr': 0.79999 }) # Empty labels and predictions result = regression_metrics.compute([], [], types.RegressionScore(), types.RegressionScore()) testing_utils.assert_dicts_almost_equal(self, result, {})
def input_spec(self) -> Spec: ret = {} ret[self.config.text_a_name] = lit_types.TextSegment() if self.config.text_b_name: ret[self.config.text_b_name] = lit_types.TextSegment() if self.is_regression: ret[self.config.label_name] = lit_types.RegressionScore( required=False) else: ret[self.config.label_name] = lit_types.CategoryLabel( required=False, vocab=self.config.labels) # The input_embs_ and grad_class fields are used for Integrated Gradients. ret["input_embs_" + self.config.text_a_name] = lit_types.TokenEmbeddings( align="tokens", required=False) if self.config.text_b_name: ret["input_embs_" + self.config.text_b_name] = lit_types.TokenEmbeddings( align="tokens", required=False) ret["grad_class"] = lit_types.CategoryLabel(required=False, vocab=self.config.labels) return ret
def label_types(): yield collections.OrderedDict([("label", lit_types.RegressionScore())])
def output_spec(self) -> lit_types.Spec: return {'score': lit_types.RegressionScore(parent='y')}
def input_spec(self) -> lit_types.Spec: return { "sentence": lit_types.TextSegment(), "label": lit_types.RegressionScore(required=False) }
def output_spec(self): return { 'proba': lit_types.RegressionScore(parent='label') # in range [0,1] }
def spec(self): return { "sentence": lit_types.TextSegment(), "label": lit_types.RegressionScore(), }
def output_spec(self): return {'score': lit_types.RegressionScore()}
def output_spec(self): return { 'pred': lit_types.RegressionScore(), 'grads': lit_types.ImageGradients(align='image'), }
def spec(self): return { 'sentence1': lit_types.TextSegment(), 'sentence2': lit_types.TextSegment(), 'label': lit_types.RegressionScore(), }