def metrics(self):
metrics = []
if self.config.predict_target:
metrics.append(
F1Metric(
prefix=const.TARGET_TAGS,
target_name=const.TARGET_TAGS,
PAD=const.PAD_TAGS_ID,
labels=const.LABELS,
)
)
metrics.append(
CorrectMetric(
prefix=const.TARGET_TAGS,
target_name=const.TARGET_TAGS,
PAD=const.PAD_TAGS_ID,
)
)
if self.config.predict_source:
metrics.append(
F1Metric(
prefix=const.SOURCE_TAGS,
target_name=const.SOURCE_TAGS,
PAD=const.PAD_TAGS_ID,
labels=const.LABELS,
)
)
metrics.append(
CorrectMetric(
prefix=const.SOURCE_TAGS,
target_name=const.SOURCE_TAGS,
PAD=const.PAD_TAGS_ID,
)
)
if self.config.predict_gaps:
metrics.append(
F1Metric(
prefix=const.GAP_TAGS,
target_name=const.GAP_TAGS,
PAD=const.PAD_TAGS_ID,
labels=const.LABELS,
)
)
metrics.append(
CorrectMetric(
prefix=const.GAP_TAGS,
target_name=const.GAP_TAGS,
PAD=const.PAD_TAGS_ID,
)
)
metrics.append(LogMetric(targets=[(const.LOSS, const.LOSS)]))
return metrics
def metrics(self):
metrics = []
main_metric = PerplexityMetric(
prefix=self.config.target_side,
target_name=self.config.target_side,
PAD=const.PAD_ID,
STOP=const.STOP_ID,
)
metrics.append(main_metric)
metrics.append(
CorrectMetric(
prefix=self.config.target_side,
target_name=self.config.target_side,
PAD=const.PAD_ID,
STOP=const.STOP_ID,
))
metrics.append(
ExpectedErrorMetric(
prefix=self.config.target_side,
target_name=self.config.target_side,
PAD=const.PAD_ID,
STOP=const.STOP_ID,
))
return metrics
def metrics(self) -> List[Metric]:
if self._metrics is None:
metrics = []
if self.config.word_level.target and self.config.word_level.gaps:
metrics += tag_metrics(
const.TARGET_TAGS,
const.GAP_TAGS,
prefix='WMT19_',
labels=self.labels(const.TARGET_TAGS),
)
if self.config.word_level.target:
metrics += tag_metrics(const.TARGET_TAGS,
labels=self.labels(const.TARGET_TAGS))
if self.config.word_level.gaps:
metrics += tag_metrics(const.GAP_TAGS,
labels=self.labels(const.GAP_TAGS))
if self.config.word_level.source:
metrics += tag_metrics(const.SOURCE_TAGS,
labels=self.labels(const.SOURCE_TAGS))
if self.config.sentence_level.hter:
metrics.append(PearsonMetric(const.SENTENCE_SCORES, prefix=''))
metrics.append(SpearmanMetric(const.SENTENCE_SCORES,
prefix=''))
metrics.append(RMSEMetric(const.SENTENCE_SCORES, prefix=''))
if self.config.sentence_level.binary:
metrics.append(
CorrectMetric(
const.BINARY,
prefix='binary_',
labels=self.labels(const.TARGET_TAGS),
))
# metrics.append(LogMetric(log_targets=[(const.LOSS, const.LOSS)]))
self._metrics = metrics
return self._metrics
def tag_metrics(*targets, prefix=None, labels=None):
metrics = [
F1MultMetric(*targets, prefix=prefix, labels=labels),
MatthewsMetric(*targets, prefix=prefix, labels=labels),
# ThresholdCalibrationMetric(*targets, prefix=prefix),
CorrectMetric(*targets, prefix=prefix, labels=labels),
]
return metrics
def metrics(self):
if self._metrics is None:
metrics = []
for output_side, layer in self.masked_word_outputs.items():
metrics.append(PerplexityMetric(output_side))
metrics.append(
ExpectedErrorMetric(output_side,
labels=self.labels(output_side)))
metrics.append(
CorrectMetric(output_side,
labels=self.labels(output_side)))
self._metrics = metrics
return self._metrics
def metrics(self):
metrics = []
if self.config.predict_target:
metrics.append(
F1Metric(
prefix=const.TARGET_TAGS,
target_name=const.TARGET_TAGS,
PAD=const.PAD_TAGS_ID,
labels=const.LABELS,
))
metrics.append(
ThresholdCalibrationMetric(
prefix=const.TARGET_TAGS,
target_name=const.TARGET_TAGS,
PAD=const.PAD_TAGS_ID,
))
metrics.append(
CorrectMetric(
prefix=const.TARGET_TAGS,
target_name=const.TARGET_TAGS,
PAD=const.PAD_TAGS_ID,
))
if self.config.predict_source:
metrics.append(
F1Metric(
prefix=const.SOURCE_TAGS,
target_name=const.SOURCE_TAGS,
PAD=const.PAD_TAGS_ID,
labels=const.LABELS,
))
metrics.append(
CorrectMetric(
prefix=const.SOURCE_TAGS,
target_name=const.SOURCE_TAGS,
PAD=const.PAD_TAGS_ID,
))
if self.config.predict_gaps:
metrics.append(
F1Metric(
prefix=const.GAP_TAGS,
target_name=const.GAP_TAGS,
PAD=const.PAD_TAGS_ID,
labels=const.LABELS,
))
metrics.append(
CorrectMetric(
prefix=const.GAP_TAGS,
target_name=const.GAP_TAGS,
PAD=const.PAD_TAGS_ID,
))
if self.config.sentence_level:
metrics.append(RMSEMetric(target_name=const.SENTENCE_SCORES))
metrics.append(PearsonMetric(target_name=const.SENTENCE_SCORES))
metrics.append(SpearmanMetric(target_name=const.SENTENCE_SCORES))
if self.config.sentence_ll:
metrics.append(
LogMetric(targets=[('model_out', const.SENT_SIGMA)]))
if self.config.binary_level:
metrics.append(
CorrectMetric(prefix=const.BINARY, target_name=const.BINARY))
if self.config.token_level and self.predictor_tgt is not None:
metrics.append(
CorrectMetric(
prefix=const.PE,
target_name=const.PE,
PAD=const.PAD_ID,
STOP=const.STOP_ID,
))
metrics.append(
ExpectedErrorMetric(
prefix=const.PE,
target_name=const.PE,
PAD=const.PAD_ID,
STOP=const.STOP_ID,
))
metrics.append(
PerplexityMetric(
prefix=const.PE,
target_name=const.PE,
PAD=const.PAD_ID,
STOP=const.STOP_ID,
))
if self.config.token_level and self.predictor_src is not None:
metrics.append(
CorrectMetric(
prefix=const.SOURCE,
target_name=const.SOURCE,
PAD=const.PAD_ID,
STOP=const.STOP_ID,
))
metrics.append(
ExpectedErrorMetric(
prefix=const.SOURCE,
target_name=const.SOURCE,
PAD=const.PAD_ID,
STOP=const.STOP_ID,
))
metrics.append(
PerplexityMetric(
prefix=const.SOURCE,
target_name=const.SOURCE,
PAD=const.PAD_ID,
STOP=const.STOP_ID,
))
metrics.append(
TokenMetric(target_name=const.TARGET,
STOP=const.STOP_ID,
PAD=const.PAD_ID))
return metrics