def output_spec(self) -> Spec:
ret = {"tokens": lit_types.Tokens()}
ret["tokens_" + self.config.text_a_name] = lit_types.Tokens()
if self.config.text_b_name:
ret["tokens_" + self.config.text_b_name] = lit_types.Tokens()
if self.is_regression:
ret["score"] = lit_types.RegressionScore(
parent=self.config.label_name)
else:
ret["probas"] = lit_types.MulticlassPreds(
parent=self.config.label_name,
vocab=self.config.labels,
null_idx=self.config.null_label_idx)
ret["cls_emb"] = lit_types.Embeddings()
# Gradients, if requested.
if self.config.compute_grads:
ret["token_grad_" +
self.config.text_a_name] = lit_types.TokenGradients(
align="tokens_" + self.config.text_a_name)
if self.config.text_b_name:
ret["token_grad_" +
self.config.text_b_name] = lit_types.TokenGradients(
align="tokens_" + self.config.text_b_name)
# Attention heads, one field for each layer.
for i in range(self.model.config.num_hidden_layers):
ret[f"layer_{i}/attention"] = lit_types.AttentionHeads(
align=("tokens", "tokens"))
return ret
def output_spec(self) -> lit_types.Spec:
return {
"tokens": lit_types.Tokens(),
"logits": lit_types.RegressionScore(),
"cls_emb": lit_types.Embeddings(),
"token_grad_sentence": lit_types.TokenGradients(align="tokens")
}
def test_find_spec_keys(self):
spec = {
"score": types.RegressionScore(),
"scalar_foo": types.Scalar(),
"text": types.TextSegment(),
"emb_0": types.Embeddings(),
"emb_1": types.Embeddings(),
"tokens": types.Tokens(),
"generated_text": types.GeneratedText(),
}
self.assertEqual(["score"], utils.find_spec_keys(spec,
types.RegressionScore))
self.assertEqual(["text", "tokens", "generated_text"],
utils.find_spec_keys(spec,
(types.TextSegment, types.Tokens)))
self.assertEqual(["emb_0", "emb_1"],
utils.find_spec_keys(spec, types.Embeddings))
self.assertEqual([], utils.find_spec_keys(spec, types.AttentionHeads))
# Check subclasses
self.assertEqual(
list(spec.keys()), utils.find_spec_keys(spec, types.LitType))
self.assertEqual(["text", "generated_text"],
utils.find_spec_keys(spec, types.TextSegment))
self.assertEqual(["score", "scalar_foo"],
utils.find_spec_keys(spec, types.Scalar))
def input_spec(self) -> Spec:
ret = {}
ret[self.config.text_a_name] = lit_types.TextSegment()
if self.config.text_b_name:
ret[self.config.text_b_name] = lit_types.TextSegment()
if self.is_regression:
ret[self.config.label_name] = lit_types.RegressionScore(required=False)
else:
ret[self.config.label_name] = lit_types.CategoryLabel(
required=False, vocab=self.config.labels)
return ret
def test_is_compatible(self):
corpusblue_metrics = metrics.CorpusBLEU()
# Only compatible with GeneratedText spec.
self.assertTrue(corpusblue_metrics.is_compatible(
types.GeneratedText()))
self.assertFalse(
corpusblue_metrics.is_compatible(
types.MulticlassPreds(vocab=[''])))
self.assertFalse(
corpusblue_metrics.is_compatible(types.RegressionScore()))
def test_is_compatible(self):
multiclass_metrics = metrics.MulticlassMetrics()
# Only compatible with MulticlassPreds spec.
self.assertTrue(
multiclass_metrics.is_compatible(
types.MulticlassPreds(vocab=[''])))
self.assertFalse(
multiclass_metrics.is_compatible(types.RegressionScore()))
self.assertFalse(
multiclass_metrics.is_compatible(types.GeneratedText()))
def set_up_pandas_dataframe_and_columns():
dataframe = pd.DataFrame.from_dict({
"feature_1": [1.0],
"feature_2": [3.0],
"label": [1.0]
})
columns = collections.OrderedDict([
("feature_1", lit_types.Scalar()),
("feature_2", lit_types.Scalar()),
("label", lit_types.RegressionScore()),
])
yield dataframe, columns
def output_spec(self) -> Spec:
ret = {"tokens": lit_types.Tokens()}
ret["tokens_" + self.config.text_a_name] = lit_types.Tokens(
parent=self.config.text_a_name)
if self.config.text_b_name:
ret["tokens_" + self.config.text_b_name] = lit_types.Tokens(
parent=self.config.text_b_name)
if self.is_regression:
ret["score"] = lit_types.RegressionScore(parent=self.config.label_name)
else:
ret["probas"] = lit_types.MulticlassPreds(
parent=self.config.label_name,
vocab=self.config.labels,
null_idx=self.config.null_label_idx)
ret["cls_emb"] = lit_types.Embeddings()
# Average embeddings, one per layer including embeddings.
for i in range(1 + self.model.config.num_hidden_layers):
ret[f"layer_{i}/avg_emb"] = lit_types.Embeddings()
ret["cls_grad"] = lit_types.Gradients(
grad_for="cls_emb", grad_target_field_key="grad_class")
# The input_embs_ and grad_class fields are used for Integrated Gradients.
ret["input_embs_" + self.config.text_a_name] = lit_types.TokenEmbeddings(
align="tokens_" + self.config.text_a_name)
if self.config.text_b_name:
ret["input_embs_" + self.config.text_b_name] = lit_types.TokenEmbeddings(
align="tokens_" + self.config.text_b_name)
# Gradients, if requested.
if self.config.compute_grads:
ret["grad_class"] = lit_types.CategoryLabel(required=False,
vocab=self.config.labels)
ret["token_grad_" + self.config.text_a_name] = lit_types.TokenGradients(
align="tokens_" + self.config.text_a_name,
grad_for="input_embs_" + self.config.text_a_name,
grad_target_field_key="grad_class")
if self.config.text_b_name:
ret["token_grad_" + self.config.text_b_name] = lit_types.TokenGradients(
align="tokens_" + self.config.text_b_name,
grad_for="input_embs_" + self.config.text_b_name,
grad_target_field_key="grad_class")
# Attention heads, one field for each layer.
for i in range(self.model.config.num_hidden_layers):
ret[f"layer_{i+1}/attention"] = lit_types.AttentionHeads(
align_in="tokens", align_out="tokens")
return ret
def test_compute(self):
regression_metrics = metrics.RegressionMetrics()
# All correct predictions.
result = regression_metrics.compute([1, 2, 3, 4], [1, 2, 3, 4],
types.RegressionScore(),
types.RegressionScore())
self.assertAlmostEqual(result, {
'mse': 0,
'pearsonr': 1.0,
'spearmanr': 1.0
})
# Some incorrect predictions.
result = regression_metrics.compute([1, 2, 3, 4], [1, 2, 5.5, 6.3],
types.RegressionScore(),
types.RegressionScore())
self.assertAlmostEqual(result, {
'mse': 2.885,
'pearsonr': 0.9656642444980974,
'spearmanr': 1.0
})
# All incorrect predictions (and not monotonic).
result = regression_metrics.compute([1, 2, 3, 4], [-5, -10, 5, 6],
types.RegressionScore(),
types.RegressionScore())
self.assertAlmostEqual(
result, {
'mse': 47.0,
'pearsonr': 0.795592252958155,
'spearmanr': 0.7999999999999999
})
# Empty labels and predictions
result = regression_metrics.compute([], [], types.RegressionScore(),
types.RegressionScore())
self.assertAlmostEqual(result, {})
def test_compute(self):
regression_metrics = metrics.RegressionMetrics()
# All correct predictions.
result = regression_metrics.compute([1, 2, 3, 4], [1, 2, 3, 4],
types.RegressionScore(),
types.RegressionScore())
testing_utils.assert_dicts_almost_equal(self, result, {
'mse': 0,
'pearsonr': 1.0,
'spearmanr': 1.0
})
# Some incorrect predictions.
result = regression_metrics.compute([1, 2, 3, 4], [1, 2, 5.5, 6.3],
types.RegressionScore(),
types.RegressionScore())
testing_utils.assert_dicts_almost_equal(self, result, {
'mse': 2.885,
'pearsonr': 0.96566,
'spearmanr': 1.0
})
# All incorrect predictions (and not monotonic).
result = regression_metrics.compute([1, 2, 3, 4], [-5, -10, 5, 6],
types.RegressionScore(),
types.RegressionScore())
testing_utils.assert_dicts_almost_equal(self, result, {
'mse': 47.0,
'pearsonr': 0.79559,
'spearmanr': 0.79999
})
# Empty labels and predictions
result = regression_metrics.compute([], [], types.RegressionScore(),
types.RegressionScore())
testing_utils.assert_dicts_almost_equal(self, result, {})
def input_spec(self) -> Spec:
ret = {}
ret[self.config.text_a_name] = lit_types.TextSegment()
if self.config.text_b_name:
ret[self.config.text_b_name] = lit_types.TextSegment()
if self.is_regression:
ret[self.config.label_name] = lit_types.RegressionScore(
required=False)
else:
ret[self.config.label_name] = lit_types.CategoryLabel(
required=False, vocab=self.config.labels)
# The input_embs_ and grad_class fields are used for Integrated Gradients.
ret["input_embs_" +
self.config.text_a_name] = lit_types.TokenEmbeddings(
align="tokens", required=False)
if self.config.text_b_name:
ret["input_embs_" +
self.config.text_b_name] = lit_types.TokenEmbeddings(
align="tokens", required=False)
ret["grad_class"] = lit_types.CategoryLabel(required=False,
vocab=self.config.labels)
return ret
def label_types():
yield collections.OrderedDict([("label", lit_types.RegressionScore())])
def output_spec(self) -> lit_types.Spec:
return {'score': lit_types.RegressionScore(parent='y')}
def input_spec(self) -> lit_types.Spec:
return {
"sentence": lit_types.TextSegment(),
"label": lit_types.RegressionScore(required=False)
}
def output_spec(self):
return {
'proba':
lit_types.RegressionScore(parent='label') # in range [0,1]
}
def spec(self):
return {
"sentence": lit_types.TextSegment(),
"label": lit_types.RegressionScore(),
}
def output_spec(self):
return {'score': lit_types.RegressionScore()}
def output_spec(self):
return {
'pred': lit_types.RegressionScore(),
'grads': lit_types.ImageGradients(align='image'),
}
def spec(self):
return {
'sentence1': lit_types.TextSegment(),
'sentence2': lit_types.TextSegment(),
'label': lit_types.RegressionScore(),
}
