def _info(self):
if self.config_name not in [
"sst2", "mnli", "mnli_mismatched", "mnli_matched", "cola",
"stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"
]:
raise KeyError(
'You should supply a configuration name selected in '
'["sst2", "mnli", "mnli_mismatched", "mnli_matched", '
'"cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans"]'
)
return nlp.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=nlp.Features({
'predictions':
nlp.Value(
'int64' if self.config_name != 'stsb' else 'float32'),
'references':
nlp.Value(
'int64' if self.config_name != 'stsb' else 'float32'),
}),
codebase_urls=[],
reference_urls=[],
format='numpy')
def _info(self):
return nlp.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=nlp.Features({
'predictions': nlp.Sequence(nlp.Value('string', id='token'), id='sequence'),
'references': nlp.Sequence(nlp.Sequence(nlp.Value('string', id='token'), id='sequence'), id='references'),
}),
codebase_urls=["https://github.com/tensorflow/nmt/blob/master/nmt/scripts/bleu.py"],
reference_urls=["https://en.wikipedia.org/wiki/BLEU",
"https://towardsdatascience.com/evaluating-text-output-in-nlp-bleu-at-your-own-risk-e8609665a213"]
)
def _info(self):
return nlp.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
homepage="https://github.com/chakki-works/seqeval",
inputs_description=_KWARGS_DESCRIPTION,
features=nlp.Features({
'predictions':
nlp.Sequence(nlp.Value('string', id='label'), id='sequence'),
'references':
nlp.Sequence(nlp.Value('string', id='label'), id='sequence'),
}),
codebase_urls=["https://github.com/chakki-works/seqeval"],
reference_urls=["https://github.com/chakki-works/seqeval"])
def _info(self):
return nlp.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
homepage="https://github.com/Tiiiger/bert_score",
inputs_description=_KWARGS_DESCRIPTION,
features=nlp.Features({
'predictions': nlp.Value('string', id='sequence'),
'references': nlp.Sequence(nlp.Value('string', id='sequence'), id='references'),
}),
codebase_urls=["https://github.com/Tiiiger/bert_score"],
reference_urls=["https://github.com/Tiiiger/bert_score",
"https://arxiv.org/abs/1904.09675"]
)
def _info(self):
return nlp.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=nlp.Features({
'predictions': nlp.Value('string', id='sequence'),
'references': nlp.Value('string', id='sequence'),
}),
codebase_urls=["https://github.com/ns-moosavi/coval"],
reference_urls=["https://github.com/ns-moosavi/coval",
"https://www.aclweb.org/anthology/P16-1060",
"http://www.conll.cemantix.org/2012/data.html"]
)
def _info(self):
return nlp.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=nlp.Features({
'predictions':
nlp.Sequence(nlp.Value('string', id='token'), id='sequence'),
'references':
nlp.Sequence(nlp.Sequence(nlp.Value('string', id='token'),
id='sequence'),
id='references'),
}),
codebase_urls=["https://github.com/cnap/gec-ranking"],
reference_urls=["https://github.com/cnap/gec-ranking"])
def _info(self):
return nlp.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=nlp.Features({
'predictions':
nlp.Value(
'int64' if self.config_name != 'sts-b' else 'float32'),
'references':
nlp.Value(
'int64' if self.config_name != 'sts-b' else 'float32'),
}),
codebase_urls=[],
reference_urls=[],
format='numpy')
def _info(self):
return nlp.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
homepage="https://github.com/google-research/bleurt",
inputs_description=_KWARGS_DESCRIPTION,
features=nlp.Features({
'predictions':
nlp.Value('string', id='sequence'),
'references':
nlp.Value('string', id='sequence'),
}),
codebase_urls=["https://github.com/google-research/bleurt"],
reference_urls=[
"https://github.com/google-research/bleurt",
"https://arxiv.org/abs/2004.04696"
])
def _info(self):
return nlp.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=nlp.Features({
'predictions':
nlp.Value('string', id='sequence'),
'references':
nlp.Value('string', id='sequence'),
}),
codebase_urls=[
"https://github.com/google-research/google-research/tree/master/rouge"
],
reference_urls=[
"https://en.wikipedia.org/wiki/ROUGE_(metric)",
"https://github.com/google-research/google-research/tree/master/rouge"
])
def _info(self):
return nlp.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=nlp.Features({
'predictions':
nlp.Value('string', id='sequence'),
'references':
nlp.Value('string', id='sequence')
}),
codebase_urls=[
"https://github.com/nltk/nltk/blob/develop/nltk/translate/meteor_score.py"
],
reference_urls=[
"https://www.nltk.org/api/nltk.translate.html#module-nltk.translate.meteor_score",
"https://en.wikipedia.org/wiki/METEOR"
])
def _info(self):
return nlp.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=nlp.Features({
'predictions': {
"id": nlp.Value("string"),
"prediction_text": nlp.Value("string")
},
'references': {
"id": nlp.Value("string"),
"answers": nlp.features.Sequence(
{"text": nlp.Value("string"), "answer_start": nlp.Value("int32"),}
),
},
}),
codebase_urls=["https://rajpurkar.github.io/SQuAD-explorer/"],
reference_urls=["https://rajpurkar.github.io/SQuAD-explorer/"]
)
def _info(self):
return nlp.MetricInfo(description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=nlp.Features({
'predictions':
nlp.Value('float', id='sequence'),
'prob_y_hat':
nlp.Value('float', id='sequence'),
'prob_y_hat_alpha':
nlp.Value('float', id='sequence'),
'null_difference':
nlp.Value('float', id='sequence'),
'model':
nlp.Value('float', id='sequence'),
'tokenizer':
nlp.Value('float', id='sequence'),
'mode':
nlp.Value('string', id='sequence'),
'normalization':
nlp.Value('bool', id='sequence'),
}))
