def forward(self, model, sample, reduce=True):
"""Compute the loss for the given sample.
Returns a tuple with three elements:
1) the loss
2) the sample size, which is used as the denominator for the gradient
3) logging outputs to display while training
"""
net_output = model(**sample['net_input'])
loss, _ = self.compute_loss(model, net_output, sample, reduce=reduce)
sample_size = sample['target'].size(
0) if self.args.sentence_avg else sample['ntokens']
# Only evalute bleu if not training
if model.training:
bleu_stats = utils.get_zero_bleu_stats()
else:
use_cuda = torch.cuda.is_available() and not self.args.cpu
if self.args.sacrebleu:
if hasattr(self.args, 'lowercase'):
scorer = bleu.SacrebleuScorer(
lowercase=self.args.lowercase)
else:
scorer = bleu.SacrebleuScorer()
else:
scorer = bleu.Scorer(self.task.target_dictionary.pad(),
self.task.target_dictionary.eos(),
self.task.target_dictionary.unk())
gen_timer = StopwatchMeter()
wps_meter = TimeMeter()
utils.run_inference_on_sample(sample=sample,
use_cuda=use_cuda,
args=self.args,
gen_timer=gen_timer,
task=self.task,
generator=self.generator,
model=model,
tgt_dict=self.task.target_dictionary,
align_dict=None,
subset=None,
src_dict=self.task.source_dictionary,
scorer=scorer,
wps_meter=wps_meter)
result_string = scorer.result_string()
bleu_stats = utils.BleuStatistics(correct=result_string.counts,
total=result_string.totals,
sys_len=result_string.sys_len,
ref_len=result_string.ref_len)
logging_output = {
'loss': utils.item(loss.data) if reduce else loss.data,
'ntokens': sample['ntokens'],
'nsentences': sample['target'].size(0),
'sample_size': sample_size,
'bleu': bleu_stats,
}
return loss, sample_size, logging_output
def aggregate_logging_outputs(logging_outputs):
"""Aggregate logging outputs from data parallel training."""
loss_sum = sum(log.get('loss', 0) for log in logging_outputs)
sample_size = sum(log.get('sample_size', 0) for log in logging_outputs)
ntokens = sum(log.get('ntokens', 0) for log in logging_outputs)
nsentences = sum(log.get('nsentences', 0) for log in logging_outputs)
downstream_loss = sum(
log.get('downstream_loss', 0) for log in logging_outputs)
bleu_stats = reduce(utils.reduce_bleu_stats, [
log.get('bleu', utils.get_zero_bleu_stats())
for log in logging_outputs
])
intermediate_loss = defaultdict(lambda: AverageMeter())
all_loss_dicts = [
log.get('intermediate_loss', defaultdict(lambda: AverageMeter()))
for log in logging_outputs
]
# Aggregate
for single_dict in all_loss_dicts:
for key, value in single_dict.items():
intermediate_loss[key] = utils.reduce_average_meter(
intermediate_loss[key], value)
# same for bleu
intermediate_bleu = defaultdict(lambda: AverageMeter())
all_bleu_dicts = [
log.get('intermediate_bleu', defaultdict(lambda: AverageMeter()))
for log in logging_outputs
]
# Aggregate
for single_dict in all_bleu_dicts:
for key, value in single_dict.items():
intermediate_bleu[key] = utils.reduce_average_meter(
intermediate_bleu[key], value)
agg_output = {
'loss': loss_sum / float(sample_size),
'sample_size': sample_size,
'ntokens': ntokens,
'nsentences': nsentences,
'downstream_loss': downstream_loss / float(sample_size),
'bleu': bleu_stats,
**intermediate_loss,
**intermediate_bleu
}
return agg_output
def aggregate_logging_outputs(logging_outputs):
"""Aggregate logging outputs from data parallel training."""
loss_sum = sum(log.get('loss', 0) for log in logging_outputs)
ntokens = sum(log.get('ntokens', 0) for log in logging_outputs)
nsentences = sum(log.get('nsentences', 0) for log in logging_outputs)
sample_size = sum(log.get('sample_size', 0) for log in logging_outputs)
bleu_stats = reduce(utils.reduce_bleu_stats, [
log.get('bleu', utils.get_zero_bleu_stats())
for log in logging_outputs
])
agg_output = {
'loss': loss_sum / sample_size / math.log(2),
'ntokens': ntokens,
'nsentences': nsentences,
'sample_size': sample_size,
'bleu': bleu_stats,
}
if sample_size != ntokens:
agg_output['nll_loss'] = loss_sum / ntokens / math.log(2)
return agg_output
def forward(self, model, sample, reduce=True):
"""Compute the loss for the given sample.
Returns a tuple with three elements:
1) the loss
2) the sample size, which is used as the denominator for the gradient
3) logging outputs to display while training
"""
loss = 0.0
downstream_loss = 0.0
bleu_stats = utils.get_zero_bleu_stats()
intermediate_loss = defaultdict(lambda: AverageMeter())
intermediate_bleu = defaultdict(lambda: AverageMeter())
for train_task in sample:
# ipdb.set_trace()
# print("| [Meta-Loss] Compute loss using {}".format(str(set(train_task.user_data_frame.task_group.values))))
single_loss, single_loss_stats = self.compute_loss(
train_task=train_task, model=model)
single_downstream_loss, single_bleu_stats, single_intermediate_loss, single_intermediate_bleu = \
single_loss_stats
loss += single_loss
downstream_loss += single_downstream_loss
bleu_stats = utils.reduce_bleu_stats(bleu_stats, single_bleu_stats)
for key, value in single_intermediate_loss.items():
intermediate_loss[key] = utils.reduce_average_meter(
intermediate_loss[key], value)
for key, value in single_intermediate_bleu.items():
intermediate_bleu[key] = utils.reduce_average_meter(
intermediate_bleu[key], value)
sample_size = len(sample)
logging_output = {
'loss': utils.item(loss.data),
'sample_size': sample_size,
'ntokens': sample_size,
'nsentences': sample_size,
'downstream_loss': downstream_loss,
'bleu': bleu_stats,
'intermediate_loss': intermediate_loss,
'intermediate_bleu': intermediate_bleu
}
return loss, sample_size, logging_output
def __init__(self):
self.correct, self.total, self.sys_len, self.ref_len = utils.get_zero_bleu_stats(
)
# TODO handle lowercase
self.scorer = bleu.SacrebleuScorer(lowercase=False)
def reset(self):
self.correct, self.total, self.sys_len, self.ref_len = utils.get_zero_bleu_stats(
)