def reduce_metrics(logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
lm_loss_sum = sum(log.get('lm_loss', 0) for log in logging_outputs)
sentence_loss_sum = sum(
log.get('sentence_loss', 0) for log in logging_outputs)
ntokens = sum(log.get('ntokens', 0) for log in logging_outputs)
nsentences = sum(log.get('nsentences', 0) for log in logging_outputs)
sample_size = sum(log.get('sample_size', 0) for log in logging_outputs)
agg_loss = sum(log.get('loss', 0) for log in logging_outputs)
metrics.log_scalar('loss',
agg_loss / sample_size /
math.log(2) if sample_size > 0 else 0.,
round=3)
metrics.log_derived(
'ppl', lambda meters: utils.get_perplexity(meters['loss'].avg))
metrics.log_scalar('lm_loss',
lm_loss_sum / ntokens /
math.log(2) if ntokens > 0 else 0.,
round=3)
metrics.log_scalar('sentence_loss',
sentence_loss_sum / nsentences /
math.log(2) if nsentences > 0 else 0.,
round=3)
metrics.log_scalar('nll_loss',
lm_loss_sum / ntokens /
math.log(2) if ntokens > 0 else 0.,
round=3)
metrics.log_scalar('ntokens', ntokens)
metrics.log_scalar('nsentences', nsentences)
metrics.log_scalar('sample_size', sample_size)
def reduce_metrics(logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training (copied from normal cross entropy)."""
loss_sum = sum(log.get("loss", 0) for log in logging_outputs)
ntokens = sum(log.get("ntokens", 0) for log in logging_outputs)
sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
metrics.log_scalar(
"loss", loss_sum / sample_size / math.log(2), sample_size, round=3
)
if sample_size != ntokens:
metrics.log_scalar(
"nll_loss", loss_sum / ntokens / math.log(2), ntokens, round=3
)
metrics.log_derived(
"ppl", lambda meters: utils.get_perplexity(meters["nll_loss"].avg)
)
else:
metrics.log_derived(
"ppl", lambda meters: utils.get_perplexity(meters["loss"].avg)
)
counts = {}
for lk in logging_outputs[0].keys():
if lk.startswith("count_"):
val = sum(log[lk] for log in logging_outputs)
metrics.log_scalar(lk, val)
counts[lk] = val
for lk in logging_outputs[0].keys():
if lk.startswith("loss_"):
val = sum(log[lk] for log in logging_outputs)
metrics.log_scalar(lk, val / sample_size / math.log(2), round=3)
elif lk.startswith("correct_"):
val = sum(log[lk] for log in logging_outputs)
metrics.log_scalar(lk, val / counts[re.sub("correct", "count", lk)])
def reduce_metrics(logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
loss_sum = utils.item(
sum(log.get("loss", 0) for log in logging_outputs))
nll_loss_sum = utils.item(
sum(log.get("nll_loss", 0) for log in logging_outputs))
alignment_loss_sum = utils.item(
sum(log.get("alignment_loss", 0) for log in logging_outputs))
ntokens = utils.item(
sum(log.get("ntokens", 0) for log in logging_outputs))
sample_size = utils.item(
sum(log.get("sample_size", 0) for log in logging_outputs))
metrics.log_scalar("loss",
loss_sum / sample_size / math.log(2),
sample_size,
round=3)
metrics.log_scalar("nll_loss",
nll_loss_sum / ntokens / math.log(2),
ntokens,
round=3)
metrics.log_scalar(
"alignment_loss",
alignment_loss_sum / math.log(2),
sample_size,
round=3,
)
metrics.log_derived(
"ppl", lambda meters: utils.get_perplexity(meters["nll_loss"].avg))
def reduce_metrics(logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
loss_sum = sum(log.get('loss', 0) for log in logging_outputs)
sample_size = sum(log.get('sample_size', 0) for log in logging_outputs)
metrics.log_scalar('loss', loss_sum / sample_size / math.log(2), sample_size, round=3)
metrics.log_derived('ppl', lambda meters: utils.get_perplexity(meters['loss'].avg))
def reduce_metrics(logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
sample_size = utils.item(
sum(log.get("sample_size", 0) for log in logging_outputs))
loss = utils.item(sum(log.get("loss", 0) for log in logging_outputs))
nll_loss = utils.item(
sum(log.get("nll_loss", 0) for log in logging_outputs))
metrics.log_scalar('loss',
loss / sample_size / math.log(2),
sample_size,
round=3)
metrics.log_scalar('nll_loss',
nll_loss / sample_size / math.log(2),
sample_size,
round=3)
metrics.log_derived(
'ppl', lambda meters: utils.get_perplexity(meters['loss'].avg))
for key in logging_outputs[0]:
if key[-5:] == "-loss":
val = sum(log.get(key, 0) for log in logging_outputs)
metrics.log_scalar(
key[:-5],
val / sample_size /
math.log(2) if sample_size > 0 else 0.0,
sample_size,
round=3,
)
def reduce_metrics(logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
loss_sum = sum(log.get('orig_loss', 0) for log in logging_outputs)
ntokens = sum(log.get('ntokens', 0) for log in logging_outputs)
sample_size = sum(log.get('sample_size', 0) for log in logging_outputs)
uid_loss_sum = sum(log.get('uid_loss', 0) for log in logging_outputs)
metrics.log_scalar('loss',
loss_sum / sample_size / math.log(2),
sample_size,
round=3)
metrics.log_scalar('uid_loss',
uid_loss_sum / sample_size / math.log(2),
sample_size,
round=3)
if sample_size != ntokens:
metrics.log_scalar('nll_loss',
loss_sum / ntokens / math.log(2),
ntokens,
round=3)
metrics.log_derived(
'ppl',
lambda meters: utils.get_perplexity(meters['nll_loss'].avg))
else:
metrics.log_derived(
'ppl', lambda meters: utils.get_perplexity(meters['loss'].avg))
def reduce_metrics(logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
loss_sum = sum(log.get("loss", 0) for log in logging_outputs)
code_loss_sum = sum(log.get("code_loss", 0) for log in logging_outputs)
value_loss_sum = sum(
log.get("value_loss", 0) for log in logging_outputs)
sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
sample_size_code = sum(
log.get("sample_size_code", 0) for log in logging_outputs)
sample_size_value = sum(
log.get("sample_size_value", 0) for log in logging_outputs)
metrics.log_scalar("loss",
loss_sum / sample_size,
sample_size,
round=3)
metrics.log_scalar("code_loss",
code_loss_sum / sample_size_code / math.log(2),
sample_size_code,
round=3)
metrics.log_scalar("valueloss",
value_loss_sum / sample_size_value,
sample_size_value,
round=3)
metrics.log_derived(
"code_ppl",
lambda meters: utils.get_perplexity(meters["code_loss"].avg))
def reduce_metrics(logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
loss_sum = utils.item(sum(log.get('loss', 0) for log in logging_outputs))
ntokens = utils.item(sum(log.get('ntokens', 0) for log in logging_outputs))
nsentences = utils.item(sum(log.get('nsentences', 0) for log in logging_outputs))
sample_size = utils.item(sum(log.get('sample_size', 0) for log in logging_outputs))
metrics.log_scalar('loss', loss_sum / sample_size / math.log(2), sample_size, round=3)
metrics.log_scalar('ntokens', ntokens)
metrics.log_scalar('nsentences', nsentences)
correct = sum(log.get("correct", 0) for log in logging_outputs)
metrics.log_scalar("_correct", correct)
total = sum(log.get("count", 0) for log in logging_outputs)
metrics.log_scalar("_total", total)
if total > 0:
metrics.log_derived(
"accuracy",
lambda meters: safe_round(meters["_correct"].sum / meters["_total"].sum, 5)
if meters["_total"].sum > 0
else float("nan"),
)
builtin_keys = {'loss', 'ntokens', 'nsentences', 'sample_size', 'correct', 'count'}
for k in logging_outputs[0]:
if k not in builtin_keys:
val = sum(log.get(k, 0) for log in logging_outputs) / len(logging_outputs)
if k.startswith('loss'):
metrics.log_scalar(k, val / sample_size / math.log(2), sample_size)
else:
metrics.log_scalar(k, val, round=3)
def reduce_metrics(self, logging_outputs, criterion):
super().reduce_metrics(logging_outputs, criterion)
bt_sample_size = sum(x.get("bt_sample_size", 0) for x in logging_outputs)
if bt_sample_size:
bt_loss_sum = sum(x.get("bt_loss", 0) for x in logging_outputs)
bt_loss_sum *= 1 / bt_sample_size / math.log(2)
metrics.log_scalar("bt_loss", bt_loss_sum, bt_sample_size, round=3)
bt_nll_loss_sum = sum(x.get("bt_nll_loss", 0) for x in logging_outputs)
bt_ntokens = sum(x.get("bt_ntokens", 0) for x in logging_outputs)
bt_nll_loss_sum *= 1 / bt_ntokens / math.log(2)
metrics.log_scalar("bt_nll_loss", bt_nll_loss_sum, bt_ntokens, round=3)
metrics.log_derived(
"bt_ppl", lambda meters: utils.get_perplexity(meters["bt_nll_loss"].avg)
)
dae_sample_size = sum(x.get("dae_sample_size", 0) for x in logging_outputs)
if dae_sample_size:
dae_loss_sum = sum(x.get("dae_loss", 0) for x in logging_outputs)
dae_loss_sum *= 1 / dae_sample_size / math.log(2)
metrics.log_scalar("dae_loss", dae_loss_sum, dae_sample_size, round=3)
dae_nll_loss_sum = sum(x.get("dae_nll_loss", 0) for x in logging_outputs)
dae_ntokens = sum(x.get("dae_ntokens", 0) for x in logging_outputs)
dae_nll_loss_sum *= 1 / dae_ntokens / math.log(2)
metrics.log_scalar("dae_nll_loss", dae_nll_loss_sum, dae_ntokens, round=3)
metrics.log_derived(
"dae_ppl",
lambda meters: utils.get_perplexity(meters["dae_nll_loss"].avg),
)
def reduce_metrics(logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
loss_sum = sum(log.get('loss', 0) for log in logging_outputs)
loss_ce = sum(log.get('ce_loss', 0) for log in logging_outputs)
loss_kd = sum(log.get('kd_loss', 0) for log in logging_outputs)
# print('debug info: loss kd =', loss_kd)
# print('debug info: loss kd =', [log.get('ce_loss', 0) for log in logging_outputs])
if 'ce_loss_teacher' in logging_outputs[0]:
loss_ce_teacher = sum(
log.get('ce_loss_teacher', 0) for log in logging_outputs)
sample_size = sum(log.get('sample_size', 0) for log in logging_outputs)
metrics.log_scalar('loss',
loss_sum / sample_size / math.log(2),
sample_size,
round=3)
metrics.log_scalar('loss_ce',
loss_ce / sample_size / math.log(2),
sample_size,
round=3)
metrics.log_scalar('loss_kd',
loss_kd / sample_size / math.log(2),
sample_size,
round=3)
if 'ce_loss_teacher' in logging_outputs[0]:
metrics.log_scalar('loss_ce_teacher',
loss_ce_teacher / sample_size / math.log(2),
sample_size,
round=3)
metrics.log_derived(
'ppl', lambda meters: utils.get_perplexity(meters['loss'].avg))
def reduce_metrics(self, logging_outputs, criterion):
super().reduce_metrics(logging_outputs, criterion)
if self.args.eval_bleu:
def sum_logs(key):
return sum(log.get(key, 0) for log in logging_outputs)
counts, totals = [], []
for i in range(EVAL_BLEU_ORDER):
counts.append(sum_logs("_bleu_counts_" + str(i)))
totals.append(sum_logs("_bleu_totals_" + str(i)))
if max(totals) > 0:
# log counts as numpy arrays -- log_scalar will sum them correctly
metrics.log_scalar("_bleu_counts", np.array(counts))
metrics.log_scalar("_bleu_totals", np.array(totals))
metrics.log_scalar("_bleu_sys_len", sum_logs("_bleu_sys_len"))
metrics.log_scalar("_bleu_ref_len", sum_logs("_bleu_ref_len"))
def compute_bleu(meters):
import inspect
import sacrebleu
fn_sig = inspect.getfullargspec(sacrebleu.compute_bleu)[0]
if "smooth_method" in fn_sig:
smooth = {"smooth_method": "exp"}
else:
smooth = {"smooth": "exp"}
bleu = sacrebleu.compute_bleu(
correct=meters["_bleu_counts"].sum,
total=meters["_bleu_totals"].sum,
sys_len=meters["_bleu_sys_len"].sum,
ref_len=meters["_bleu_ref_len"].sum,
**smooth)
return round(bleu.score, 2)
metrics.log_derived("bleu", compute_bleu)
def reduce_metrics(logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
loss_sum = sum(log.get('loss', 0) for log in logging_outputs)
loss_sum_random = sum(
log.get('loss_random', 0) for log in logging_outputs)
loss_sum_max = sum(log.get('loss_max', 0) for log in logging_outputs)
nll_loss_sum = sum(log.get('nll_loss', 0) for log in logging_outputs)
ntokens = sum(log.get('ntokens', 0) for log in logging_outputs)
sample_size = sum(log.get('sample_size', 0) for log in logging_outputs)
metrics.log_scalar('loss',
loss_sum / sample_size / math.log(2),
sample_size,
round=3)
metrics.log_scalar('nll_loss',
nll_loss_sum / ntokens / math.log(2),
ntokens,
round=3)
metrics.log_derived('ppl',
lambda meters: round(2**meters['nll_loss'].avg, 3))
metrics.log_scalar('loss_random',
loss_sum_random / sample_size / math.log(2),
sample_size,
round=4)
metrics.log_scalar('loss_max',
loss_sum_max / sample_size / math.log(2),
sample_size,
round=5)
def reduce_metrics(logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
loss_sum = sum(log.get('loss', 0) for log in logging_outputs)
nll_loss_sum = sum(log.get('nll_loss', 0) for log in logging_outputs)
ntokens = sum(log.get('ntokens', 0) for log in logging_outputs)
sample_size = sum(log.get('sample_size', 0) for log in logging_outputs)
#ori_loss=utils.item(sum(log.get('ori_loss', 0) for log in logging_outputs))
#Dist_M_sum=utils.item(sum(log.get('Dist_M_sum', 0).float() for log in logging_outputs))
#vec_en2de_mean_norm=utils.item(sum(log.get('vec_en2de_mean_norm', 0).float() for log in logging_outputs))
#vec_de2en_mean_norm=utils.item(sum(log.get('vec_de2en_mean_norm', 0).float() for log in logging_outputs))
#metrics.log_scalar('ori_loss', loss_sum , sample_size, round=3)
#metrics.log_scalar('Dist_M_sum', Dist_M_sum , sample_size, round=3)
#metrics.log_scalar('vec_en2de_mean_norm', vec_en2de_mean_norm , sample_size, round=3)
#metrics.log_scalar('vec_de2en_mean_norm', vec_de2en_mean_norm , sample_size, round=3)
metrics.log_scalar('loss',
loss_sum / sample_size / math.log(2),
sample_size,
round=3)
metrics.log_scalar('nll_loss',
nll_loss_sum / ntokens / math.log(2),
ntokens,
round=3)
metrics.log_derived(
'ppl', lambda meters: utils.get_perplexity(meters['nll_loss'].avg))
def reduce_metrics(cls, logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
loss_sum = sum(log.get('loss', 0) for log in logging_outputs)
nll_loss_sum = sum(log.get('nll_loss', 0) for log in logging_outputs)
ntokens = sum(log.get('ntokens', 0) for log in logging_outputs)
sample_size = sum(log.get('sample_size', 0) for log in logging_outputs)
metrics.log_scalar('loss',
loss_sum / sample_size / math.log(2),
sample_size,
round=3)
metrics.log_scalar('nll_loss',
nll_loss_sum / ntokens / math.log(2),
ntokens,
round=3)
metrics.log_derived(
'ppl', lambda meters: utils.get_perplexity(meters['nll_loss'].avg))
total = utils.item(sum(log.get('total', 0) for log in logging_outputs))
if total > 0:
metrics.log_scalar('total', total)
n_correct = utils.item(
sum(log.get('n_correct', 0) for log in logging_outputs))
metrics.log_scalar('n_correct', n_correct)
metrics.log_derived(
'accuracy',
lambda meters: round(
meters['n_correct'].sum * 100.0 / meters['total'].sum, 3)
if meters['total'].sum > 0 else float('nan'),
)
def reduce_metrics(cls, logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
loss_sum = sum(log.get("loss", 0) for log in logging_outputs)
nll_loss_sum = sum(log.get("nll_loss", 0) for log in logging_outputs)
ntokens = sum(log.get("ntokens", 0) for log in logging_outputs)
sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
metrics.log_scalar(
"loss", loss_sum / sample_size / math.log(2), sample_size, round=3
)
metrics.log_scalar(
"nll_loss", nll_loss_sum / ntokens / math.log(2), ntokens, round=3
)
metrics.log_derived(
"ppl", lambda meters: utils.get_perplexity(meters["nll_loss"].avg)
)
total = utils.item(sum(log.get("total", 0) for log in logging_outputs))
if total > 0:
metrics.log_scalar("total", total)
n_correct = utils.item(
sum(log.get("n_correct", 0) for log in logging_outputs)
)
metrics.log_scalar("n_correct", n_correct)
metrics.log_derived(
"accuracy",
lambda meters: round(
meters["n_correct"].sum * 100.0 / meters["total"].sum, 3
)
if meters["total"].sum > 0
else float("nan"),
)
def reduce_metrics(logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
# loss_sum = sum(log.get('loss', 0) for log in logging_outputs)
#sample_size = sum(log.get('sample_size', 0) for log in logging_ouRtputs)
#metrics.log_scalar('loss', loss_sum / sample_size / math.log(2), sample_size, round=3)
loss_mask = sum(log.get('loss_mask', 0) for log in logging_outputs)
loss_decode = sum(log.get('loss_decode', 0) for log in logging_outputs)
sample_size_decode = sum(log.get('sample_size_decode', 0) for log in logging_outputs)
sample_size_mask = sum(log.get('sample_size_mask', 0) for log in logging_outputs)
accumulate_step = sum(log.get('sample_size_t', 0) for log in logging_outputs)
loss_decode=loss_decode / sample_size_decode / math.log(2)
loss_mask=loss_mask / sample_size_mask / math.log(2)
metrics.log_scalar('loss_decode', loss_decode, sample_size_decode, round=3)
metrics.log_scalar('loss_mask', loss_mask, sample_size_mask, round=3)
loss_sum=0.5*loss_mask+0.5*loss_decode
metrics.log_scalar('loss', loss_sum , 0.5*sample_size_decode+0.5*sample_size_mask, round=3)
metrics.log_derived('ppl', lambda meters: utils.get_perplexity(meters['loss'].avg))
metrics.log_scalar('sample_size_t', accumulate_step , accumulate_step, round=3)
token = sum(log.get('ntokens', 0) for log in logging_outputs)
metrics.log_scalar('ntokens', token , token, round=3)
metrics.log_scalar('sample_size_decode', sample_size_decode , sample_size_decode, round=3)
sample_size = sum(log.get('sample_size', 0) for log in logging_outputs)
metrics.log_scalar('sample_size', sample_size , sample_size, round=3)
def reduce_metrics(logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
loss_sum = sum(log.get('loss', 0) for log in logging_outputs)
nll_loss_sum = sum(log.get('nll_loss', 0) for log in logging_outputs)
ctrl_loss_sum = sum(log.get('ctrl_loss', 0) for log in logging_outputs)
sel_entropy = sum(log.get('sel_entropy', 0) for log in logging_outputs)
batch_entropy = sum(
log.get('batch_entropy', 0) for log in logging_outputs)
ctrl_entropy_ratio = sel_entropy / (batch_entropy + _EPS)
ntokens = sum(log.get('ntokens', 0) for log in logging_outputs)
sample_size = sum(log.get('sample_size', 0) for log in logging_outputs)
nsentences = sum(log.get('nsentences', 0) for log in logging_outputs)
metrics.log_scalar('loss',
loss_sum / sample_size / math.log(2),
sample_size,
round=3)
metrics.log_scalar('nll_loss',
nll_loss_sum / ntokens / math.log(2),
ntokens,
round=3)
metrics.log_scalar('ctrl_loss',
ctrl_loss_sum / sample_size / math.log(2),
nsentences,
round=3)
metrics.log_scalar('sel_entropy', sel_entropy, 1, round=3)
metrics.log_scalar('batch_entropy', batch_entropy, 1, round=3)
metrics.log_scalar('ctrl_entropy_ratio',
ctrl_entropy_ratio,
1,
round=3)
metrics.log_derived(
'ppl', lambda meters: utils.get_perplexity(meters['nll_loss'].avg))
def reduce_metrics(logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
loss_sum = sum(log.get('loss', 0) for log in logging_outputs)
nll_loss_sum = sum(log.get('nll_loss', 0) for log in logging_outputs)
ntokens = sum(log.get('ntokens', 0) for log in logging_outputs)
sample_size = sum(log.get('sample_size', 0) for log in logging_outputs)
nsentences = sum(log.get('nsentences', 0) for log in logging_outputs)
metrics.log_scalar('loss',
loss_sum / sample_size / math.log(2),
sample_size,
round=3,
priority=3)
metrics.log_scalar('nll_loss_s',
nll_loss_sum / nsentences,
nsentences,
round=3,
priority=4)
metrics.log_scalar('nll_loss_t',
nll_loss_sum / ntokens / math.log(2),
ntokens,
round=3,
priority=5)
metrics.log_derived(
'ppl',
lambda meters: utils.get_perplexity(meters['nll_loss_t'].avg),
priority=6)
def reduce_metrics(logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
loss_sum = utils.item(sum(log.get('loss', 0) for log in logging_outputs))
ntokens = utils.item(sum(log.get('ntokens', 0) for log in logging_outputs))
sample_size = utils.item(sum(log.get('sample_size', 0) for log in logging_outputs))
#batch_size=len([log.get('ori_loss', 0) for log in logging_outputs])
#print (str(logging_outputs))
ori_loss=utils.item(sum(log.get('ori_loss', 0) for log in logging_outputs))
Dist_M_sum=utils.item(sum(log.get('Dist_M_sum', 0).float() for log in logging_outputs))
vec_en2de_mean_norm=utils.item(sum(log.get('vec_en2de_mean_norm', 0).float() for log in logging_outputs))
vec_de2en_mean_norm=utils.item(sum(log.get('vec_de2en_mean_norm', 0).float() for log in logging_outputs))
#print ('batch_size = {}, sample_size = {}, ori_loss = {}, Dist_M_sum = {} vec_en2de_mean_norm = {}, vec_de2en_mean_norm = {}'.format(batch_size, sample_size,ori_loss,Dist_M_sum,vec_en2de_mean_norm,vec_de2en_mean_norm))
#print ('ori_loss= {} Dist_M_sum ={} vec_en2de_mean_norm = {}, vec_de2en_mean_norm ={}'.format(ori_loss,Dist_M_sum,vec_en2de_mean_norm,vec_de2en_mean_norm))
metrics.log_scalar('loss', loss_sum / sample_size / math.log(2), sample_size, round=3)
metrics.log_scalar('ori_loss', loss_sum , sample_size, round=3)
metrics.log_scalar('Dist_M_sum', Dist_M_sum , sample_size, round=3)
metrics.log_scalar('vec_en2de_mean_norm', vec_en2de_mean_norm , sample_size, round=3)
metrics.log_scalar('vec_de2en_mean_norm', vec_de2en_mean_norm , sample_size, round=3)
if sample_size != ntokens:
metrics.log_scalar('nll_loss', loss_sum / ntokens / math.log(2), ntokens, round=3)
metrics.log_derived('ppl', lambda meters: utils.get_perplexity(meters['nll_loss'].avg))
else:
metrics.log_derived('ppl', lambda meters: utils.get_perplexity(meters['loss'].avg))
def reduce_metrics(logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
loss_sum = utils.item(
sum(log.get('loss', 0) for log in logging_outputs))
nll_loss_sum = utils.item(
sum(log.get('nll_loss', 0) for log in logging_outputs))
regularization_loss_sum = utils.item(
sum(log.get('regularization_loss', 0) for log in logging_outputs))
ntokens = utils.item(
sum(log.get('ntokens', 0) for log in logging_outputs))
sample_size = utils.item(
sum(log.get('sample_size', 0) for log in logging_outputs))
metrics.log_scalar('loss',
loss_sum / sample_size / math.log(2),
sample_size,
round=3)
metrics.log_scalar('nll_loss',
nll_loss_sum / ntokens / math.log(2),
ntokens,
round=3)
metrics.log_scalar('regularization_loss',
regularization_loss_sum / sample_size,
sample_size,
round=3)
metrics.log_derived(
'ppl', lambda meters: utils.get_perplexity(meters['nll_loss'].avg))
def reduce_metrics(logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
loss_sum = sum(log.get("loss", 0) for log in logging_outputs)
nll_loss_sum = sum(log.get("nll_loss", 0) for log in logging_outputs)
ntokens = sum(log.get("ntokens", 0) for log in logging_outputs)
sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
symm_kl_sum = sum(log.get("symm_kl", 0) for log in logging_outputs)
self_kl_sum = sum(log.get("self_kl", 0) for log in logging_outputs)
self_cv_sum = sum(log.get("self_cv", 0) for log in logging_outputs)
metrics.log_scalar("symm_kl", symm_kl_sum /
sample_size, sample_size, round=3)
metrics.log_scalar("self_kl", self_kl_sum /
sample_size, sample_size, round=3)
metrics.log_scalar("self_cv", self_cv_sum /
sample_size, sample_size, round=3)
metrics.log_scalar(
"loss", loss_sum / sample_size / math.log(2), sample_size, round=3
)
metrics.log_scalar(
"nll_loss", nll_loss_sum / ntokens / math.log(2), ntokens, round=3
)
metrics.log_derived(
"ppl", lambda meters: utils.get_perplexity(meters["nll_loss"].avg)
)
def reduce_metrics(logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
loss_sum = sum(log.get('loss', 0) for log in logging_outputs)
nll_loss_sum = sum(log.get('nll_loss', 0) for log in logging_outputs)
critic_score_sum = sum(
log.get('critic_score', 0) for log in logging_outputs)
ntokens = sum(log.get('ntokens', 0) for log in logging_outputs)
sample_size = sum(log.get('sample_size', 0) for log in logging_outputs)
metrics.log_scalar('loss',
loss_sum / sample_size / math.log(2),
sample_size,
round=3)
metrics.log_scalar('nll_loss',
nll_loss_sum / ntokens / math.log(2),
ntokens,
round=3)
metrics.log_scalar('critic_score',
critic_score_sum / sample_size / math.log(2),
sample_size)
metrics.log_derived(
'ppl', lambda meters: utils.get_perplexity(meters['nll_loss'].avg))
if 'reward' in logging_outputs[0].keys():
reward_sum = sum(log.get('reward', 0) for log in logging_outputs)
metrics.log_scalar('reward',
reward_sum / sample_size / math.log(2),
sample_size,
round=3)
def reduce_metrics(cls, logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
loss_sum = sum(log.get("loss", 0) for log in logging_outputs)
nll_loss_sum = sum(log.get("nll_loss", 0) for log in logging_outputs)
ctc_loss_sum = sum(log.get("ctc_loss", 0) for log in logging_outputs)
ntokens = sum(log.get("ntokens", 0) for log in logging_outputs)
sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
if 'nhit' in logging_outputs[0] and \
'ntokens_masked' in logging_outputs[0]:
nhit = sum(log['nhit'] for log in logging_outputs)
ntokens_masked = sum(log['ntokens_masked']
for log in logging_outputs)
assert nhit <= ntokens_masked
if ntokens_masked > 0:
hit_rate = nhit / ntokens_masked
else:
hit_rate = -1
#TODO: check how to fill the 3 arguments below
metrics.log_scalar("nhit", nhit, round=3, weight=0)
metrics.log_scalar("ntokens_masked",
ntokens_masked,
round=3,
weight=0)
metrics.log_scalar("hit_rate", hit_rate, round=3, weight=0)
# May have to adjust below for CTC loss as well
metrics.log_scalar("loss",
loss_sum / sample_size / math.log(2),
sample_size,
round=3)
metrics.log_scalar("nll_loss",
nll_loss_sum / ntokens / math.log(2),
ntokens,
round=3)
metrics.log_scalar("ctc_loss",
ctc_loss_sum / ntokens / math.log(2),
ntokens,
round=3)
metrics.log_derived(
"ppl", lambda meters: utils.get_perplexity(meters["nll_loss"].avg))
total = utils.item(sum(log.get("total", 0) for log in logging_outputs))
if total > 0:
metrics.log_scalar("total", total)
n_correct = utils.item(
sum(log.get("n_correct", 0) for log in logging_outputs))
metrics.log_scalar("n_correct", n_correct)
metrics.log_derived(
"accuracy",
lambda meters: round(
meters["n_correct"].sum * 100.0 / meters["total"].sum, 3)
if meters["total"].sum > 0 else float("nan"),
)
def reduce_metrics(cls, logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
loss_sum = sum(log.get("loss", 0) for log in logging_outputs)
nll_loss_sum = sum(log.get("nll_loss", 0) for log in logging_outputs)
ntokens = sum(log.get("ntokens", 0) for log in logging_outputs)
sample_size = sum(log.get("sample_size", 0) for log in logging_outputs)
classifier_loss_sum = sum(
log.get("classifier_loss", 0) for log in logging_outputs)
metrics.log_scalar("loss",
loss_sum / sample_size / math.log(2),
sample_size,
round=3)
metrics.log_scalar("nll_loss",
nll_loss_sum / ntokens / math.log(2),
ntokens,
round=3)
metrics.log_derived(
"ppl", lambda meters: utils.get_perplexity(meters["nll_loss"].avg))
metrics.log_scalar("classifier_loss",
classifier_loss_sum / ntokens / math.log(2),
ntokens,
round=3)
total = utils.item(sum(log.get("total", 0) for log in logging_outputs))
if total > 0:
metrics.log_scalar("total", total)
n_correct = utils.item(
sum(log.get("n_correct", 0) for log in logging_outputs))
metrics.log_scalar("n_correct", n_correct)
metrics.log_derived(
"accuracy",
lambda meters: round(
meters["n_correct"].sum * 100.0 / meters["total"].sum, 3)
if meters["total"].sum > 0 else float("nan"),
)
t = logging_outputs[0].get("per_lang_n_correct", None)
if t is not None:
for lang_id in t:
n_correct = sum(
log.get("per_lang_n_correct").get(lang_id, 0)
for log in logging_outputs)
n_total = sum(
log.get("per_lang_total").get(lang_id, 0)
for log in logging_outputs)
metrics.log_scalar(
f"accuracy_lang_{lang_id}",
round(n_correct * 100.0 /
n_total, 3) if n_total > 0 else float("nan"),
priority=100,
)
metrics.log_scalar(
f"n_total_lang_{lang_id}",
n_total,
priority=100,
)
def reduce_metrics(self, logging_outputs, criterion):
super().reduce_metrics(logging_outputs, criterion)
if self.args.eval_bleu:
def sum_logs(key):
return sum(log.get(key, 0) for log in logging_outputs)
counts, totals = [], []
for i in range(EVAL_BLEU_ORDER):
counts.append(sum_logs('_bleu_counts_' + str(i)))
totals.append(sum_logs('_bleu_totals_' + str(i)))
if max(totals) > 0:
# log counts as numpy arrays -- log_scalar will sum them correctly
metrics.log_scalar('_bleu_counts', np.array(counts))
metrics.log_scalar('_bleu_totals', np.array(totals))
metrics.log_scalar('_bleu_sys_len', sum_logs('_bleu_sys_len'))
metrics.log_scalar('_bleu_ref_len', sum_logs('_bleu_ref_len'))
def compute_bleu(meters):
import inspect
import sacrebleu
fn_sig = inspect.getfullargspec(sacrebleu.compute_bleu)[0]
if 'smooth_method' in fn_sig:
smooth = {'smooth_method': 'exp'}
else:
smooth = {'smooth': 'exp'}
bleu = sacrebleu.compute_bleu(
correct=meters['_bleu_counts'].sum,
total=meters['_bleu_totals'].sum,
sys_len=meters['_bleu_sys_len'].sum,
ref_len=meters['_bleu_ref_len'].sum,
**smooth)
return round(bleu.score, 2)
metrics.log_derived('bleu', compute_bleu)
for coder in ['encoder']:
label = coder + '_ctrl_probs'
if any([(label not in log) for log in logging_outputs]):
continue
def compute_batch_entropy(meters):
probs = meters['_' + label]
return (-probs.val * np.log(probs.val + _EPS)).mean()
probs = torch.cat([log[label].cpu() for log in logging_outputs], 0)
sel_entropy = (-probs * torch.log(probs + _EPS)).mean(-1)
metrics.log_scalar('_' + label, np.array(probs.mean(0)))
metrics.log_scalar(coder + '_ctrl_sel_entropy',
np.array(sel_entropy.mean()))
metrics.log_derived(coder + '_ctrl_batch_entropy',
compute_batch_entropy)
def reduce_metrics(logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
loss_sum = utils.item(
sum(log.get("loss", 0) for log in logging_outputs))
ntokens = utils.item(
sum(log.get("ntokens", 0) for log in logging_outputs))
nsentences = utils.item(
sum(log.get("nsentences", 0) for log in logging_outputs))
sample_size = utils.item(
sum(log.get("sample_size", 0) for log in logging_outputs))
metrics.log_scalar("loss",
loss_sum / sample_size / math.log(2),
sample_size,
round=3)
metrics.log_scalar("ntokens", ntokens)
metrics.log_scalar("nsentences", nsentences)
correct = sum(log.get("correct", 0) for log in logging_outputs)
metrics.log_scalar("_correct", correct)
total = sum(log.get("count", 0) for log in logging_outputs)
metrics.log_scalar("_total", total)
if total > 0:
metrics.log_derived(
"accuracy",
lambda meters: safe_round(
meters["_correct"].sum / meters["_total"].sum, 5)
if meters["_total"].sum > 0 else float("nan"),
)
builtin_keys = {
"loss",
"ntokens",
"nsentences",
"sample_size",
"correct",
"count",
}
handled_keys = reduce_probe_metrics(logging_outputs, metrics)
builtin_keys.update(handled_keys)
for k in logging_outputs[0]:
if k not in builtin_keys:
val = sum(log.get(k, 0) for log in logging_outputs)
if k.startswith("loss"):
metrics.log_scalar(k,
val / sample_size / math.log(2),
sample_size,
round=3)
else:
metrics.log_scalar(k, val / len(logging_outputs), round=3)
def reduce_metrics(self, logging_outputs, criterion):
super().reduce_metrics(logging_outputs, criterion)
if self.cfg.eval_bleu:
def sum_logs(key):
import torch
result = sum(log.get(key, 0) for log in logging_outputs)
if torch.is_tensor(result):
result = result.cpu()
return result
counts, totals = [], []
for i in range(EVAL_BLEU_ORDER):
counts.append(sum_logs("_bleu_counts_" + str(i)))
totals.append(sum_logs("_bleu_totals_" + str(i)))
if max(totals) > 0:
# log counts as numpy arrays -- log_scalar will sum them correctly
metrics.log_scalar("_bleu_counts", np.array(counts))
metrics.log_scalar("_bleu_totals", np.array(totals))
metrics.log_scalar("_bleu_sys_len", sum_logs("_bleu_sys_len"))
metrics.log_scalar("_bleu_ref_len", sum_logs("_bleu_ref_len"))
def compute_bleu(meters):
import inspect
try:
from sacrebleu.metrics import BLEU
comp_bleu = BLEU.compute_bleu
except ImportError:
# compatibility API for sacrebleu 1.x
import sacrebleu
comp_bleu = sacrebleu.compute_bleu
fn_sig = inspect.getfullargspec(comp_bleu)[0]
if "smooth_method" in fn_sig:
smooth = {"smooth_method": "exp"}
else:
smooth = {"smooth": "exp"}
bleu = comp_bleu(
correct=meters["_bleu_counts"].sum,
total=meters["_bleu_totals"].sum,
sys_len=meters["_bleu_sys_len"].sum,
ref_len=meters["_bleu_ref_len"].sum,
**smooth,
)
return round(bleu.score, 2)
metrics.log_derived("bleu", compute_bleu)
def reduce_metrics(logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
loss_sum = utils.item(
sum(log.get("loss", 0) for log in logging_outputs))
ctc_loss_sum = utils.item(
sum(log.get("ctc_loss", 0) for log in logging_outputs))
ce_loss_sum = utils.item(
sum(log.get("ce_loss", 0) for log in logging_outputs))
ntokens = utils.item(
sum(log.get("ntokens", 0) for log in logging_outputs))
correct_sum = sum(log.get("correct", 0) for log in logging_outputs)
total_sum = sum(log.get("total", 0) for log in logging_outputs)
nsentences = utils.item(
sum(log.get("nsentences", 0) for log in logging_outputs))
sample_size = utils.item(
sum(log.get("sample_size", 0) for log in logging_outputs))
metrics.log_scalar("loss",
loss_sum / sample_size / math.log(2),
sample_size,
round=3)
metrics.log_scalar("ctc_loss",
ctc_loss_sum / sample_size / math.log(2),
sample_size,
round=3)
metrics.log_scalar("ce_loss",
ce_loss_sum / sample_size / math.log(2),
sample_size,
round=3)
metrics.log_scalar(
"acc", correct_sum * 100.0 / total_sum if total_sum > 0 else 0.0)
metrics.log_scalar("ntokens", ntokens)
metrics.log_scalar("nsentences", nsentences)
if sample_size != ntokens:
metrics.log_scalar("nll_loss",
loss_sum / ntokens / math.log(2),
ntokens,
round=3)
c_errors = sum(log.get("c_errors", 0) for log in logging_outputs)
metrics.log_scalar("_c_errors", c_errors)
c_total = sum(log.get("c_total", 0) for log in logging_outputs)
metrics.log_scalar("_c_total", c_total)
if c_total > 0:
metrics.log_derived(
"uer",
lambda meters: safe_round(
meters["_c_errors"].sum * 100.0 / meters["_c_total"].sum, 3
) if meters["_c_total"].sum > 0 else float("nan"),
)
def reduce_metrics(logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
total_loss_sum = sum(log.get('total_loss', 0) for log in logging_outputs)
loss_sum = sum(log.get('loss', 0) for log in logging_outputs)
nll_loss_sum = sum(log.get('nll_loss', 0) for log in logging_outputs)
ntokens = sum(log.get('ntokens', 0) for log in logging_outputs)
sample_size = sum(log.get('sample_size', 0) for log in logging_outputs)
cor_pearson = sum(log.get('pearson', 0) for log in logging_outputs)
metrics.log_scalar('total_loss', total_loss_sum / sample_size / math.log(2), sample_size, round=3)
metrics.log_scalar('loss', loss_sum / sample_size / math.log(2), sample_size, round=3)
metrics.log_scalar('nll_loss', nll_loss_sum / ntokens / math.log(2), ntokens, round=3)
metrics.log_scalar('pearson', cor_pearson, round=3)
metrics.log_derived('ppl', lambda meters: utils.get_perplexity(meters['nll_loss'].avg))
def reduce_metrics(logging_outputs) -> None:
"""Aggregate logging outputs from data parallel training."""
loss_sum = sum(log.get('loss', 0) for log in logging_outputs)
masker_loss_sum = sum(
log.get('masker_loss', 0) for log in logging_outputs)
total_loss_sum = sum(
log.get('total_loss', 0) for log in logging_outputs)
masker_entropy = sum(
log.get('masker_entropy', 0) for log in logging_outputs)
top2_dist = sum(log.get('top2_dist', 0)
for log in logging_outputs) / len(logging_outputs)
top5_dist = sum(log.get('top5_dist', 0)
for log in logging_outputs) / len(logging_outputs)
weight_mean = sum(
log.get('weight_mean', 0)
for log in logging_outputs) / len(logging_outputs)
batch_mean = sum(log.get('batch_mean', 0)
for log in logging_outputs) / len(logging_outputs)
#print(len(logging_outputs), 'len logging outputs')
sample_size = sum(log.get('sample_size', 0) for log in logging_outputs)
nsentences = sum(log.get('nsentences', 0) for log in logging_outputs)
metrics.log_scalar('loss',
loss_sum / sample_size / math.log(2),
sample_size,
round=3)
metrics.log_scalar('masker_entropy',
masker_entropy / nsentences,
8,
round=3)
metrics.log_scalar('top2_dist', top2_dist / nsentences, 32, round=4)
metrics.log_scalar('top5_dist', top5_dist / nsentences, 32, round=4)
metrics.log_scalar('weight_mean',
weight_mean / nsentences,
32,
round=2)
metrics.log_scalar('batch_mean', batch_mean / nsentences, 32, round=2)
#metrics.log_scalar('nsentences', nsentences, 1, round=1)
metrics.log_scalar('masker_loss',
masker_loss_sum / sample_size / math.log(2),
sample_size,
round=5)
metrics.log_scalar('total_loss',
total_loss_sum / sample_size / math.log(2),
sample_size,
round=3)
metrics.log_derived('ppl',
lambda meters: round(2**meters['loss'].avg, 3))
