def _info(self): supported_configs = [ "sst2", "mnli", "mnli_mismatched", "mnli_matched", "cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans", "cb", "boolq" ] config_name = self.config_name if config_name.startswith('few_'): config_name = config_name[4:] if config_name not in supported_configs: raise KeyError( f"You should supply a configuration name selected in {supported_configs}" ) return datasets.MetricInfo( description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features({ "predictions": datasets.Value( "int64" if config_name != "stsb" else "float32"), "references": datasets.Value( "int64" if config_name != "stsb" else "float32"), }), codebase_urls=[], reference_urls=[], format="numpy", )
def _info(self): if self.config_name not in [ "boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg", ]: raise KeyError( "You should supply a configuration name selected in " '["boolq", "cb", "copa", "multirc", "record", "rte", "wic", "wsc", "wsc.fixed", "axb", "axg",]' ) return datasets.MetricInfo( description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features(self._get_feature_types()), codebase_urls=[], reference_urls=[], format="numpy" if not self.config_name == "record" and not self.config_name == "multirc" else None, )
def _info(self): return datasets.MetricInfo( description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features({ "predictions": { "id": datasets.Value("string"), "prediction_text": datasets.features.Sequence(datasets.Value("string")), }, "references": { "id": datasets.Value("string"), "answers": datasets.features.Sequence({ "text": datasets.Value("string"), "answer_start": datasets.Value("int32"), }), }, }), codebase_urls=["https://www.atticusprojectai.org/cuad"], reference_urls=["https://www.atticusprojectai.org/cuad"], )
def _info(self): return datasets.MetricInfo( description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features({ "predictions": datasets.Value("string", id="sequence"), "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), }), codebase_urls=[ "https://github.com/huggingface/transformers/blob/master/src/transformers/data/metrics/squad_metrics.py", "https://github.com/cocoxu/simplification/blob/master/SARI.py", "https://github.com/tensorflow/tensor2tensor/blob/master/tensor2tensor/utils/sari_hook.py", "https://github.com/mjpost/sacreBLEU", ], reference_urls=[ "https://www.aclweb.org/anthology/Q16-1029.pdf", "https://github.com/mjpost/sacreBLEU", "https://en.wikipedia.org/wiki/BLEU", "https://towardsdatascience.com/evaluating-text-output-in-nlp-bleu-at-your-own-risk-e8609665a213", ], )
def _info(self): return datasets.MetricInfo( description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features({ "predictions": { "id": datasets.Value("string"), "prediction_text": datasets.Value("string") }, "references": { "id": datasets.Value("string"), "answers": datasets.features.Sequence({ "text": datasets.Value("string"), "answer_start": datasets.Value("int32"), }), }, }), codebase_urls=["https://rajpurkar.github.io/SQuAD-explorer/"], reference_urls=["https://rajpurkar.github.io/SQuAD-explorer/"], )
def _info(self): assert self.config_name in [ "sst2", "mnli", "mnli_mismatched", "mnli_matched", "cola", "stsb", "mrpc", "qqp", "qnli", "rte", "wnli", "hans", ] return ds.MetricInfo( description="", citation="", inputs_description="", features=ds.Features( { "predictions": ds.Value("int64" if self.config_name != "stsb" else "float32"), "references": ds.Value("int64" if self.config_name != "stsb" else "float32"), } ), codebase_urls=[], reference_urls=[], format="numpy", )
def _info(self): return datasets.MetricInfo( description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features(self._get_feature_types()), reference_urls=[ "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.mean_squared_error.html" ], )
def _info(self): return datasets.MetricInfo( description="_DESCRIPTION", citation="_CITATION", inputs_description="_KWARGS_DESCRIPTION", features=datasets.Features({ "predictions": datasets.Value("int32"), "references": datasets.Value("int32"), }), reference_urls=[""], )
def _info(self): return datasets.MetricInfo( description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features({ "X": datasets.Sequence(datasets.Value("float", id="sequence"), id="X"), }), )
def _info(self): return ds.MetricInfo( description="", citation="", inputs_description="", features=ds.Features({ "predictions": ds.Value("string", id="sequence"), "references": ds.Value("string", id="sequence"), }), )
def _info(self): return datasets.MetricInfo( description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features({ "input_texts": datasets.Value("string"), }), reference_urls=[ "https://huggingface.co/docs/transformers/perplexity" ], )
def _info(self): return datasets.MetricInfo( description=_DESCRIPTION, inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features({ "predictions": datasets.Value("int"), "references": datasets.Value("int"), }), reference_urls=[ "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.recall_score.html" ], )
def _info(self): return datasets.MetricInfo( description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features( { "predictions": datasets.Value("string"), "references": datasets.Value("string"), } ), homepage="https://github.com/moussaKam/FrugalScore", )
def _info(self): return datasets.MetricInfo( description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features({ "predictions": datasets.Value("string", id="sequence"), "references": datasets.Value("string", id="sequence"), }), reference_urls=[], )
def _info(self): return datasets.MetricInfo( description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features( { "predictions": datasets.Value("float"), "references": datasets.Value("float"), } ), reference_urls=["https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.pearsonr.html"], )
def _info(self): return ds.MetricInfo( description="", citation="", inputs_description="", features=ds.Features({ "predictions": ds.Sequence(ds.Value("int32")), "references": ds.Sequence(ds.Value("int32")), } if self.config_name == "multilabel" else { "predictions": ds.Value("int32"), "references": ds.Value("int32"), }), )
def _info(self): return datasets.MetricInfo( description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features( { "predictions": datasets.Sequence(datasets.Value("int8")), "references": datasets.Sequence(datasets.Value("int8")), } ), codebase_urls=[""], reference_urls=[""], )
def _info(self): return datasets.MetricInfo( description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features({ "predictions": datasets.Value("string"), "references": datasets.Value("string"), }), # Homepage of the metric for documentation homepage="https://github.com/hendrycks/math", # Additional links to the codebase or references codebase_urls=["https://github.com/hendrycks/math"], )
def _info(self): return datasets.MetricInfo( description=_DESCRIPTION, citation=_CITATION, homepage="https://github.com/Tiiiger/bert_score", inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features( { "predictions": datasets.Value("string", id="sequence"), "references": datasets.Sequence(datasets.Value("string", id="sequence"), id="references"), } ), codebase_urls=["https://github.com/Tiiiger/bert_score"], reference_urls=["https://github.com/Tiiiger/bert_score", "https://arxiv.org/abs/1904.09675"], )
def _info(self): return datasets.MetricInfo( description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features( { "predictions": datasets.Value("float32"), "references": datasets.Value("float32"), } ), codebase_urls=[], reference_urls=[], format="numpy", )
def _info(self): return ds.MetricInfo( description="", citation="", inputs_description="", features=ds.Features({ "predictions": ds.Value( "int64" if self.config_name != "sts-b" else "float32"), "references": ds.Value( "int64" if self.config_name != "sts-b" else "float32"), }), format="numpy", )
def _info(self): return datasets.MetricInfo( description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features( { "predictions": datasets.Value("string", id="sequence"), "references": datasets.Value("string", id="sequence"), } ), codebase_urls=["https://github.com/jitsi/jiwer/"], reference_urls=[ "https://en.wikipedia.org/wiki/Word_error_rate", ], )
def _info(self): return datasets.MetricInfo( description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features({ 'predictions': datasets.Value( 'int64' if self.config_name != 'sts-b' else 'float32'), 'references': datasets.Value( 'int64' if self.config_name != 'sts-b' else 'float32'), }), codebase_urls=[], reference_urls=[], format='numpy')
def _info(self): return datasets.MetricInfo( description=_DESCRIPTION, citation=_CITATION, homepage="https://github.com/google-research/bleurt", inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features( { "predictions": datasets.Value("string", id="sequence"), "references": datasets.Value("string", id="sequence"), } ), codebase_urls=["https://github.com/google-research/bleurt"], reference_urls=["https://github.com/google-research/bleurt", "https://arxiv.org/abs/2004.04696"], )
def _info(self): # 会作为 datasets.MetricInfo 的信息 return datasets.MetricInfo( # 这是将在metric页面上显示的描述。 description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, # 定义预测和真实标签的格式, 注意预测时的标签格式,一般为int格式, 如果是回归模型为float32 features=datasets.Features({ 'predictions': datasets.Value("int64"), 'references': datasets.Value("int64"), }), homepage="http://metric.homepage", #其它介绍信息 codebase_urls=["http://github.com/path/to/codebase/of/new_metric"], reference_urls=["http://path.to.reference.url/new_metric"])
def _info(self): return datasets.MetricInfo( description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features({ "predictions": datasets.Sequence(datasets.Value("int8")), "references": datasets.Sequence(datasets.Value("int8")), }), codebase_urls=[ "https://github.com/shrimai/Topological-Sort-for-Sentence-Ordering/blob/master/topological_sort.py#L91-L105" ], reference_urls=["https://www.aclweb.org/anthology/J06-4002/"], )
def _info(self): return datasets.MetricInfo( description=_DESCRIPTION, citation=_CITATION, homepage="https://github.com/chakki-works/seqeval", inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features({ "predictions": datasets.Sequence(datasets.Value("string", id="label"), id="sequence"), "references": datasets.Sequence(datasets.Value("string", id="label"), id="sequence"), }), codebase_urls=["https://github.com/chakki-works/seqeval"], reference_urls=["https://github.com/chakki-works/seqeval"], )
def _info(self): return datasets.MetricInfo( description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features( { "predictions": datasets.Value("string", id="sequence"), "references": datasets.Value("string", id="sequence"), } ), codebase_urls=["https://github.com/google-research/google-research/tree/master/rouge"], reference_urls=[ "https://en.wikipedia.org/wiki/ROUGE_(metric)", "https://github.com/google-research/google-research/tree/master/rouge", ], )
def _info(self): return datasets.MetricInfo( description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, features=datasets.Features( { "predictions": datasets.Sequence(datasets.Value("int32")), "references": datasets.Sequence(datasets.Value("int32")), } if self.config_name == "multilabel" else { "predictions": datasets.Value("int32"), "references": datasets.Value("int32"), }), reference_urls=[ "https://scikit-learn.org/stable/modules/generated/sklearn.metrics.accuracy_score.html" ], )
def _info(self): # TODO: Specifies the datasets.MetricInfo object return datasets.MetricInfo( # This is the description that will appear on the metrics page. description=_DESCRIPTION, citation=_CITATION, inputs_description=_KWARGS_DESCRIPTION, # This defines the format of each prediction and reference features=datasets.Features({ 'predictions': datasets.Value('string'), 'references': datasets.Value('string'), }), # Homepage of the metric for documentation homepage="http://metric.homepage", # Additional links to the codebase or references codebase_urls=["http://github.com/path/to/codebase/of/new_metric"], reference_urls=["http://path.to.reference.url/new_metric"])