def reduce_metrics(logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
loss_sum = utils.item(
sum(log.get('loss', 0) for log in logging_outputs))
nll_loss_sum = utils.item(
sum(log.get('nll_loss', 0) for log in logging_outputs))
alignment_loss_sum = utils.item(
sum(log.get('alignment_loss', 0) for log in logging_outputs))
ntokens = utils.item(
sum(log.get('ntokens', 0) for log in logging_outputs))
sample_size = utils.item(
sum(log.get('sample_size', 0) for log in logging_outputs))
metrics.log_scalar('loss',
loss_sum / sample_size / math.log(2),
sample_size,
round=3)
metrics.log_scalar('nll_loss',
nll_loss_sum / ntokens / math.log(2),
ntokens,
round=3)
metrics.log_scalar('alignment_loss',
alignment_loss_sum / sample_size / math.log(2),
sample_size,
round=3)
metrics.log_derived(
'ppl', lambda meters: utils.get_perplexity(meters['nll_loss'].avg))
def reduce_metrics(logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
loss_sum = sum(log.get('loss', 0) for log in logging_outputs)
ntokens = sum(log.get('ntokens', 0) for log in logging_outputs)
sample_size = sum(log.get('sample_size', 0) for log in logging_outputs)
match = sum(log.get('match', 0) for log in logging_outputs)
total = sum(log.get('total', 0) for log in logging_outputs)
metrics.log_scalar('match', value=match, round=0)
metrics.log_scalar('total', value=total, round=0)
metrics.log_scalar('loss',
loss_sum / sample_size / math.log(2),
sample_size,
round=3)
if sample_size != ntokens:
metrics.log_scalar('nll_loss',
loss_sum / ntokens / math.log(2),
ntokens,
round=3)
metrics.log_derived(
'ppl',
lambda meters: utils.get_perplexity(meters['nll_loss'].avg))
else:
metrics.log_derived(
'ppl', lambda meters: utils.get_perplexity(meters['loss'].avg))
def reduce_metrics(logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
loss_sum = sum(log.get('loss', 0) for log in logging_outputs)
sample_size = sum(log.get('sample_size', 0) for log in logging_outputs)
# metrics.log_scalar('loss', loss_sum / sample_size / math.log(2), sample_size, round=3)
metrics.log_scalar('loss',
loss_sum / sample_size,
sample_size,
round=3)
metrics.log_derived(
'ppl', lambda meters: utils.get_perplexity(meters['loss'].avg))
def reduce_metrics(self, logging_outputs, criterion):
super().reduce_metrics(logging_outputs, criterion)
if self.args['task']['eval_bleu']:
if self.args['task']['eval_with_sacrebleu']:
def sum_logs(key):
import torch
result = sum(log.get(key, 0) for log in logging_outputs)
if torch.is_tensor(result):
result = result.cpu()
return result
counts, totals = [], []
for i in range(EVAL_BLEU_ORDER):
counts.append(sum_logs('_bleu_counts_' + str(i)))
totals.append(sum_logs('_bleu_totals_' + str(i)))
if max(totals) > 0:
# log counts as numpy arrays -- log_scalar will sum them correctly
metrics.log_scalar('_bleu_counts', np.array(counts))
metrics.log_scalar('_bleu_totals', np.array(totals))
metrics.log_scalar('_bleu_sys_len',
sum_logs('_bleu_sys_len'))
metrics.log_scalar('_bleu_ref_len',
sum_logs('_bleu_ref_len'))
def compute_bleu(meters):
import inspect
import sacrebleu
fn_sig = inspect.getfullargspec(
sacrebleu.compute_bleu)[0]
if 'smooth_method' in fn_sig:
smooth = {'smooth_method': 'exp'}
else:
smooth = {'smooth': 'exp'}
bleu = sacrebleu.compute_bleu(
correct=meters['_bleu_counts'].sum,
total=meters['_bleu_totals'].sum,
sys_len=meters['_bleu_sys_len'].sum,
ref_len=meters['_bleu_ref_len'].sum,
**smooth)
return round(bleu.score, 6)
metrics.log_derived('bleu', compute_bleu)
else:
def sum_logs(key):
return sum(log.get(key, 0) for log in logging_outputs)
metrics.log_scalar('bleu', sum_logs('bleu'), round=6)