def reduce_metrics(logging_outputs) -> None: """Aggregate logging outputs from data parallel training.""" loss_sum = utils.item( sum(log.get('loss', 0) for log in logging_outputs)) nll_loss_sum = utils.item( sum(log.get('nll_loss', 0) for log in logging_outputs)) alignment_loss_sum = utils.item( sum(log.get('alignment_loss', 0) for log in logging_outputs)) ntokens = utils.item( sum(log.get('ntokens', 0) for log in logging_outputs)) sample_size = utils.item( sum(log.get('sample_size', 0) for log in logging_outputs)) metrics.log_scalar('loss', loss_sum / sample_size / math.log(2), sample_size, round=3) metrics.log_scalar('nll_loss', nll_loss_sum / ntokens / math.log(2), ntokens, round=3) metrics.log_scalar('alignment_loss', alignment_loss_sum / sample_size / math.log(2), sample_size, round=3) metrics.log_derived( 'ppl', lambda meters: utils.get_perplexity(meters['nll_loss'].avg))
def reduce_metrics(logging_outputs) -> None: """Aggregate logging outputs from data parallel training.""" loss_sum = sum(log.get('loss', 0) for log in logging_outputs) ntokens = sum(log.get('ntokens', 0) for log in logging_outputs) sample_size = sum(log.get('sample_size', 0) for log in logging_outputs) match = sum(log.get('match', 0) for log in logging_outputs) total = sum(log.get('total', 0) for log in logging_outputs) metrics.log_scalar('match', value=match, round=0) metrics.log_scalar('total', value=total, round=0) metrics.log_scalar('loss', loss_sum / sample_size / math.log(2), sample_size, round=3) if sample_size != ntokens: metrics.log_scalar('nll_loss', loss_sum / ntokens / math.log(2), ntokens, round=3) metrics.log_derived( 'ppl', lambda meters: utils.get_perplexity(meters['nll_loss'].avg)) else: metrics.log_derived( 'ppl', lambda meters: utils.get_perplexity(meters['loss'].avg))
def reduce_metrics(logging_outputs) -> None: """Aggregate logging outputs from data parallel training.""" loss_sum = sum(log.get('loss', 0) for log in logging_outputs) sample_size = sum(log.get('sample_size', 0) for log in logging_outputs) # metrics.log_scalar('loss', loss_sum / sample_size / math.log(2), sample_size, round=3) metrics.log_scalar('loss', loss_sum / sample_size, sample_size, round=3) metrics.log_derived( 'ppl', lambda meters: utils.get_perplexity(meters['loss'].avg))
def reduce_metrics(self, logging_outputs, criterion): super().reduce_metrics(logging_outputs, criterion) if self.args['task']['eval_bleu']: if self.args['task']['eval_with_sacrebleu']: def sum_logs(key): import torch result = sum(log.get(key, 0) for log in logging_outputs) if torch.is_tensor(result): result = result.cpu() return result counts, totals = [], [] for i in range(EVAL_BLEU_ORDER): counts.append(sum_logs('_bleu_counts_' + str(i))) totals.append(sum_logs('_bleu_totals_' + str(i))) if max(totals) > 0: # log counts as numpy arrays -- log_scalar will sum them correctly metrics.log_scalar('_bleu_counts', np.array(counts)) metrics.log_scalar('_bleu_totals', np.array(totals)) metrics.log_scalar('_bleu_sys_len', sum_logs('_bleu_sys_len')) metrics.log_scalar('_bleu_ref_len', sum_logs('_bleu_ref_len')) def compute_bleu(meters): import inspect import sacrebleu fn_sig = inspect.getfullargspec( sacrebleu.compute_bleu)[0] if 'smooth_method' in fn_sig: smooth = {'smooth_method': 'exp'} else: smooth = {'smooth': 'exp'} bleu = sacrebleu.compute_bleu( correct=meters['_bleu_counts'].sum, total=meters['_bleu_totals'].sum, sys_len=meters['_bleu_sys_len'].sum, ref_len=meters['_bleu_ref_len'].sum, **smooth) return round(bleu.score, 6) metrics.log_derived('bleu', compute_bleu) else: def sum_logs(key): return sum(log.get(key, 0) for log in logging_outputs) metrics.log_scalar('bleu', sum_logs('bleu'), round=6)