def _info(self):
supported_configs = [
"sst2", "mnli", "mnli_mismatched", "mnli_matched", "cola", "stsb",
"mrpc", "qqp", "qnli", "rte", "wnli", "hans", "cb", "boolq"
]
config_name = self.config_name
if config_name.startswith('few_'):
config_name = config_name[4:]
if config_name not in supported_configs:
raise KeyError(
f"You should supply a configuration name selected in {supported_configs}"
)
return datasets.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=datasets.Features({
"predictions":
datasets.Value(
"int64" if config_name != "stsb" else "float32"),
"references":
datasets.Value(
"int64" if config_name != "stsb" else "float32"),
}),
codebase_urls=[],
reference_urls=[],
format="numpy",
)
def _info(self):
if self.config_name not in [
"boolq",
"cb",
"copa",
"multirc",
"record",
"rte",
"wic",
"wsc",
"wsc.fixed",
"axb",
"axg",
]:
raise KeyError(
"You should supply a configuration name selected in "
'["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]'
)
return datasets.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=datasets.Features(self._get_feature_types()),
codebase_urls=[],
reference_urls=[],
format="numpy" if not self.config_name == "record"
and not self.config_name == "multirc" else None,
)
def _info(self):
return datasets.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=datasets.Features({
"predictions": {
"id":
datasets.Value("string"),
"prediction_text":
datasets.features.Sequence(datasets.Value("string")),
},
"references": {
"id":
datasets.Value("string"),
"answers":
datasets.features.Sequence({
"text":
datasets.Value("string"),
"answer_start":
datasets.Value("int32"),
}),
},
}),
codebase_urls=["https://www.atticusprojectai.org/cuad"],
reference_urls=["https://www.atticusprojectai.org/cuad"],
)
def _info(self):
return datasets.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=datasets.Features({
"predictions":
datasets.Value("string", id="sequence"),
"references":
datasets.Sequence(datasets.Value("string", id="sequence"),
id="references"),
}),
codebase_urls=[
"https://github.com/huggingface/transformers/blob/master/src/transformers/data/metrics/squad_metrics.py",
"https://github.com/cocoxu/simplification/blob/master/SARI.py",
"https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/sari_hook.py",
"https://github.com/mjpost/sacreBLEU",
],
reference_urls=[
"https://www.aclweb.org/anthology/Q16-1029.pdf",
"https://github.com/mjpost/sacreBLEU",
"https://en.wikipedia.org/wiki/BLEU",
"https://towardsdatascience.com/evaluating-text-output-in-nlp-bleu-at-your-own-risk-e8609665a213",
],
)
def _info(self):
return datasets.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=datasets.Features({
"predictions": {
"id": datasets.Value("string"),
"prediction_text": datasets.Value("string")
},
"references": {
"id":
datasets.Value("string"),
"answers":
datasets.features.Sequence({
"text":
datasets.Value("string"),
"answer_start":
datasets.Value("int32"),
}),
},
}),
codebase_urls=["https://rajpurkar.github.io/SQuAD-explorer/"],
reference_urls=["https://rajpurkar.github.io/SQuAD-explorer/"],
)
def _info(self):
assert self.config_name in [
"sst2",
"mnli",
"mnli_mismatched",
"mnli_matched",
"cola",
"stsb",
"mrpc",
"qqp",
"qnli",
"rte",
"wnli",
"hans",
]
return ds.MetricInfo(
description="",
citation="",
inputs_description="",
features=ds.Features(
{
"predictions": ds.Value("int64" if self.config_name != "stsb" else "float32"),
"references": ds.Value("int64" if self.config_name != "stsb" else "float32"),
}
),
codebase_urls=[],
reference_urls=[],
format="numpy",
)
def _info(self):
return datasets.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=datasets.Features(self._get_feature_types()),
reference_urls=[
"https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_squared_error.html"
],
)
def _info(self):
return datasets.MetricInfo(
description="_DESCRIPTION",
citation="_CITATION",
inputs_description="_KWARGS_DESCRIPTION",
features=datasets.Features({
"predictions": datasets.Value("int32"),
"references": datasets.Value("int32"),
}),
reference_urls=[""],
)
def _info(self):
return datasets.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=datasets.Features({
"X":
datasets.Sequence(datasets.Value("float", id="sequence"),
id="X"),
}),
)
def _info(self):
return ds.MetricInfo(
description="",
citation="",
inputs_description="",
features=ds.Features({
"predictions":
ds.Value("string", id="sequence"),
"references":
ds.Value("string", id="sequence"),
}),
)
def _info(self):
return datasets.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=datasets.Features({
"input_texts": datasets.Value("string"),
}),
reference_urls=[
"https://huggingface.co/docs/transformers/perplexity"
],
)
def _info(self):
return datasets.MetricInfo(
description=_DESCRIPTION,
inputs_description=_KWARGS_DESCRIPTION,
features=datasets.Features({
"predictions": datasets.Value("int"),
"references": datasets.Value("int"),
}),
reference_urls=[
"https://scikit-learn.org/stable/modules/generated/sklearn.metrics.recall_score.html"
],
)
def _info(self):
return datasets.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=datasets.Features(
{
"predictions": datasets.Value("string"),
"references": datasets.Value("string"),
}
),
homepage="https://github.com/moussaKam/FrugalScore",
)
def _info(self):
return datasets.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=datasets.Features({
"predictions":
datasets.Value("string", id="sequence"),
"references":
datasets.Value("string", id="sequence"),
}),
reference_urls=[],
)
def _info(self):
return datasets.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=datasets.Features(
{
"predictions": datasets.Value("float"),
"references": datasets.Value("float"),
}
),
reference_urls=["https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.pearsonr.html"],
)
def _info(self):
return ds.MetricInfo(
description="",
citation="",
inputs_description="",
features=ds.Features({
"predictions": ds.Sequence(ds.Value("int32")),
"references": ds.Sequence(ds.Value("int32")),
} if self.config_name == "multilabel" else {
"predictions": ds.Value("int32"),
"references": ds.Value("int32"),
}),
)
def _info(self):
return datasets.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=datasets.Features(
{
"predictions": datasets.Sequence(datasets.Value("int8")),
"references": datasets.Sequence(datasets.Value("int8")),
}
),
codebase_urls=[""],
reference_urls=[""],
)
def _info(self):
return datasets.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=datasets.Features({
"predictions": datasets.Value("string"),
"references": datasets.Value("string"),
}),
# Homepage of the metric for documentation
homepage="https://github.com/hendrycks/math",
# Additional links to the codebase or references
codebase_urls=["https://github.com/hendrycks/math"],
)
def _info(self):
return datasets.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
homepage="https://github.com/Tiiiger/bert_score",
inputs_description=_KWARGS_DESCRIPTION,
features=datasets.Features(
{
"predictions": datasets.Value("string", id="sequence"),
"references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"),
}
),
codebase_urls=["https://github.com/Tiiiger/bert_score"],
reference_urls=["https://github.com/Tiiiger/bert_score", "https://arxiv.org/abs/1904.09675"],
)
def _info(self):
return datasets.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=datasets.Features(
{
"predictions": datasets.Value("float32"),
"references": datasets.Value("float32"),
}
),
codebase_urls=[],
reference_urls=[],
format="numpy",
)
def _info(self):
return ds.MetricInfo(
description="",
citation="",
inputs_description="",
features=ds.Features({
"predictions":
ds.Value(
"int64" if self.config_name != "sts-b" else "float32"),
"references":
ds.Value(
"int64" if self.config_name != "sts-b" else "float32"),
}),
format="numpy",
)
def _info(self):
return datasets.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=datasets.Features(
{
"predictions": datasets.Value("string", id="sequence"),
"references": datasets.Value("string", id="sequence"),
}
),
codebase_urls=["https://github.com/jitsi/jiwer/"],
reference_urls=[
"https://en.wikipedia.org/wiki/Word_error_rate",
],
)
def _info(self):
return datasets.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=datasets.Features({
'predictions':
datasets.Value(
'int64' if self.config_name != 'sts-b' else 'float32'),
'references':
datasets.Value(
'int64' if self.config_name != 'sts-b' else 'float32'),
}),
codebase_urls=[],
reference_urls=[],
format='numpy')
def _info(self):
return datasets.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
homepage="https://github.com/google-research/bleurt",
inputs_description=_KWARGS_DESCRIPTION,
features=datasets.Features(
{
"predictions": datasets.Value("string", id="sequence"),
"references": datasets.Value("string", id="sequence"),
}
),
codebase_urls=["https://github.com/google-research/bleurt"],
reference_urls=["https://github.com/google-research/bleurt", "https://arxiv.org/abs/2004.04696"],
)
def _info(self):
# 会作为 datasets.MetricInfo 的信息
return datasets.MetricInfo(
# 这是将在metric页面上显示的描述。
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
# 定义预测和真实标签的格式, 注意预测时的标签格式，一般为int格式, 如果是回归模型为float32
features=datasets.Features({
'predictions': datasets.Value("int64"),
'references': datasets.Value("int64"),
}),
homepage="http://metric.homepage",
#其它介绍信息
codebase_urls=["http://github.com/path/to/codebase/of/new_metric"],
reference_urls=["http://path.to.reference.url/new_metric"])
def _info(self):
return datasets.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=datasets.Features({
"predictions":
datasets.Sequence(datasets.Value("int8")),
"references":
datasets.Sequence(datasets.Value("int8")),
}),
codebase_urls=[
"https://github.com/shrimai/Topological-Sort-for-Sentence-Ordering/blob/master/topological_sort.py#L91-L105"
],
reference_urls=["https://www.aclweb.org/anthology/J06-4002/"],
)
def _info(self):
return datasets.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
homepage="https://github.com/chakki-works/seqeval",
inputs_description=_KWARGS_DESCRIPTION,
features=datasets.Features({
"predictions":
datasets.Sequence(datasets.Value("string", id="label"),
id="sequence"),
"references":
datasets.Sequence(datasets.Value("string", id="label"),
id="sequence"),
}),
codebase_urls=["https://github.com/chakki-works/seqeval"],
reference_urls=["https://github.com/chakki-works/seqeval"],
)
def _info(self):
return datasets.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=datasets.Features(
{
"predictions": datasets.Value("string", id="sequence"),
"references": datasets.Value("string", id="sequence"),
}
),
codebase_urls=["https://github.com/google-research/google-research/tree/master/rouge"],
reference_urls=[
"https://en.wikipedia.org/wiki/ROUGE_(metric)",
"https://github.com/google-research/google-research/tree/master/rouge",
],
)
def _info(self):
return datasets.MetricInfo(
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
features=datasets.Features(
{
"predictions": datasets.Sequence(datasets.Value("int32")),
"references": datasets.Sequence(datasets.Value("int32")),
} if self.config_name == "multilabel" else {
"predictions": datasets.Value("int32"),
"references": datasets.Value("int32"),
}),
reference_urls=[
"https://scikit-learn.org/stable/modules/generated/sklearn.metrics.accuracy_score.html"
],
)
def _info(self):
# TODO: Specifies the datasets.MetricInfo object
return datasets.MetricInfo(
# This is the description that will appear on the metrics page.
description=_DESCRIPTION,
citation=_CITATION,
inputs_description=_KWARGS_DESCRIPTION,
# This defines the format of each prediction and reference
features=datasets.Features({
'predictions': datasets.Value('string'),
'references': datasets.Value('string'),
}),
# Homepage of the metric for documentation
homepage="http://metric.homepage",
# Additional links to the codebase or references
codebase_urls=["http://github.com/path/to/codebase/of/new_metric"],
reference_urls=["http://path.to.reference.url/new_metric"])
