def validate(args, epoch, trainer, criterion, dataset, subset, ngpus): """Evaluate the model on the validation set and return the average loss.""" itr = dataset.dataloader(subset, batch_size=None, max_tokens=args.max_tokens, max_positions=args.max_positions, skip_invalid_size_inputs_valid_test=args. skip_invalid_size_inputs_valid_test) loss_meter = AverageMeter() desc = '| epoch {:03d} | valid on \'{}\' subset'.format(epoch, subset) with progress_bar(itr, desc, leave=False) as t: for _, sample in data.skip_group_enumerator(t, ngpus): ntokens = sum(s['ntokens'] for s in sample) loss = trainer.valid_step(sample, criterion) loss_meter.update(loss, ntokens) t.set_postfix(loss='{:.2f}'.format(loss_meter.avg), refresh=False) val_loss = loss_meter.avg t.write(desc + ' | valid loss {:2.2f} | valid ppl {:3.2f}'.format( val_loss, math.pow(2, val_loss))) # update and return the learning rate return val_loss
def train(args, epoch, batch_offset, trainer, criterion, dataset, num_gpus): """Train the model for one epoch.""" itr = dataset.dataloader( args.train_subset, num_workers=args.workers, max_tokens=args.max_tokens, seed=args.seed, epoch=epoch, max_positions=args.max_positions, sample_without_replacement=args.sample_without_replacement, skip_invalid_size_inputs_valid_test=args. skip_invalid_size_inputs_valid_test) loss_meter = AverageMeter() bsz_meter = AverageMeter() # sentences per batch wpb_meter = AverageMeter() # words per batch wps_meter = TimeMeter() # words per second clip_meter = AverageMeter() # % of updates clipped gnorm_meter = AverageMeter() # gradient norm desc = '| epoch {:03d}'.format(epoch) lr = trainer.get_lr() with progress_bar(itr, desc, leave=False) as t: for i, sample in data.skip_group_enumerator(t, num_gpus, batch_offset): loss, grad_norm = trainer.train_step(sample, criterion) ntokens = sum(s['ntokens'] for s in sample) src_size = sum(s['src_tokens'].size(0) for s in sample) loss_meter.update(loss, ntokens) bsz_meter.update(src_size) wpb_meter.update(ntokens) wps_meter.update(ntokens) clip_meter.update(1 if grad_norm > args.clip_norm else 0) gnorm_meter.update(grad_norm) t.set_postfix(collections.OrderedDict([ ('loss', '{:.2f} ({:.2f})'.format(loss, loss_meter.avg)), ('wps', '{:5d}'.format(round(wps_meter.avg))), ('wpb', '{:5d}'.format(round(wpb_meter.avg))), ('bsz', '{:5d}'.format(round(bsz_meter.avg))), ('lr', lr), ('clip', '{:3.0f}%'.format(clip_meter.avg * 100)), ('gnorm', '{:.4f}'.format(gnorm_meter.avg)), ]), refresh=False) if i == 0: # ignore the first mini-batch in words-per-second calculation wps_meter.reset() if args.save_interval > 0 and (i + 1) % args.save_interval == 0: trainer.save_checkpoint(args, epoch, i + 1) fmt = desc + ' | train loss {:2.2f} | train ppl {:3.2f}' fmt += ' | s/checkpoint {:7d} | words/s {:6d} | words/batch {:6d}' fmt += ' | bsz {:5d} | lr {:0.6f} | clip {:3.0f}% | gnorm {:.4f}' t.write( fmt.format(loss_meter.avg, math.pow(2, loss_meter.avg), round(wps_meter.elapsed_time), round(wps_meter.avg), round(wpb_meter.avg), round(bsz_meter.avg), lr, clip_meter.avg * 100, gnorm_meter.avg))
def validate(args, epoch, trainer, dataset, max_positions, subset): """Evaluate the model on the validation set and return the average loss.""" itr = dataset.eval_dataloader( subset, max_tokens=args.max_tokens, max_sentences=args.max_sentences_valid, max_positions=max_positions, skip_invalid_size_inputs_valid_test=args.skip_invalid_size_inputs_valid_test, descending=True, # largest batch first to warm the caching allocator ) loss_meter = AverageMeter() nll_loss_meter = AverageMeter() extra_meters = collections.defaultdict(lambda: AverageMeter()) prefix = 'valid on \'{}\' subset'.format(subset) with utils.build_progress_bar(args, itr, epoch, prefix) as t: for _, sample in data.skip_group_enumerator(t, args.num_gpus): loss_dict = trainer.valid_step(sample) ntokens = sum(s['ntokens'] for s in sample) loss = loss_dict['loss'] del loss_dict['loss'] # don't include in extra_meters or extra_postfix if 'nll_loss' in loss_dict: nll_loss = loss_dict['nll_loss'] nll_loss_meter.update(nll_loss, ntokens) loss_meter.update(loss, ntokens) extra_postfix = [] for k, v in loss_dict.items(): extra_meters[k].update(v) extra_postfix.append((k, extra_meters[k].avg)) t.log(collections.OrderedDict([ ('valid loss', round(loss_meter.avg, 2)), ] + extra_postfix)) t.print(collections.OrderedDict([ ('valid loss', round(loss_meter.avg, 2)), ('valid ppl', get_perplexity(nll_loss_meter.avg if nll_loss_meter.count > 0 else loss_meter.avg)), ] + [ (k, meter.avg) for k, meter in extra_meters.items() ])) # update and return the learning rate return loss_meter.avg
def validate(args, epoch, trainer, dataset, subset, ngpus): """Evaluate the model on the validation set and return the average loss.""" itr = dataset.dataloader(subset, batch_size=None, max_tokens=args.max_tokens, max_positions=args.max_positions, skip_invalid_size_inputs_valid_test=args. skip_invalid_size_inputs_valid_test) loss_meter = AverageMeter() extra_meters = collections.defaultdict(lambda: AverageMeter()) desc = '| epoch {:03d} | valid on \'{}\' subset'.format(epoch, subset) with progress_bar(itr, desc, leave=False) as t: for _, sample in data.skip_group_enumerator(t, ngpus): loss_dict = trainer.valid_step(sample) loss = loss_dict['loss'] del loss_dict[ 'loss'] # don't include in extra_meters or extra_postfix ntokens = sum(s['ntokens'] for s in sample) loss_meter.update(loss, ntokens) extra_postfix = [] for k, v in loss_dict.items(): extra_meters[k].update(v) extra_postfix.append((k, '{:.4f}'.format(extra_meters[k].avg))) t.set_postfix(collections.OrderedDict([ ('loss', '{:.2f}'.format(loss_meter.avg)), ] + extra_postfix), refresh=False) val_loss = loss_meter.avg fmt = desc + ' | valid loss {:2.2f} | valid ppl {:3.2f}'.format( val_loss, get_perplexity(val_loss)) fmt += ''.join(' | {} {:.4f}'.format(k, meter.avg) for k, meter in extra_meters.items()) t.write(fmt) # update and return the learning rate return val_loss
def validate(args, epoch, trainer, dataset, max_positions, subset, ngpus): """Evaluate the model on the validation set and return the average loss.""" itr = dataset.eval_dataloader( subset, max_tokens=args.max_tokens, max_sentences=args.max_sentences, max_positions=max_positions, skip_invalid_size_inputs_valid_test=args.skip_invalid_size_inputs_valid_test, descending=True, # largest batch first to warm the caching allocator ) loss_meter = AverageMeter() extra_meters = collections.defaultdict(lambda: AverageMeter()) prefix = 'valid on \'{}\' subset'.format(subset) with utils.build_progress_bar(args, itr, epoch, prefix) as t: for _, sample in data.skip_group_enumerator(t, ngpus): loss_dict = trainer.valid_step(sample) loss = loss_dict['loss'] del loss_dict['loss'] # don't include in extra_meters or extra_postfix ntokens = sum(s['ntokens'] for s in sample) loss_meter.update(loss, ntokens) extra_postfix = [] for k, v in loss_dict.items(): extra_meters[k].update(v) extra_postfix.append((k, extra_meters[k].avg)) t.log(collections.OrderedDict([ ('valid loss', round(loss_meter.avg, 2)), ] + extra_postfix)) t.print(collections.OrderedDict([ ('valid loss', round(loss_meter.avg, 2)), ('valid ppl', get_perplexity(loss_meter.avg)), ] + [ (k, meter.avg) for k, meter in extra_meters.items() ])) # update and return the learning rate return loss_meter.avg
def validate(args, epoch, trainer, criterion, dataset, subset, ngpus): """Evaluate the model on the validation set and return the average loss.""" itr = dataset.dataloader(subset, batch_size=None, max_tokens=args.max_tokens, max_positions=args.max_positions) loss_meter = AverageMeter() rouge_greedy_meter = AverageMeter() rouge_sampled_meter = AverageMeter() desc = '| epoch {:03d} | valid on \'{}\' subset'.format(epoch, subset) with progress_bar(itr, desc, leave=False) as t: for _, sample in data.skip_group_enumerator(t, ngpus): ntokens = sum(s['ntokens'] for s in sample) loss, mean_rouge_greedy, mean_rouge_sampled = trainer.valid_step( sample, criterion) loss_meter.update(loss, ntokens) rouge_greedy_meter.update(mean_rouge_greedy, 1) rouge_sampled_meter.update(mean_rouge_sampled, 1) t.set_postfix( collections.OrderedDict([ ('loss', '{:.2f}'.format(loss_meter.avg)), ('ROUGE-L/f (greedy)', '{:.4f}'.format(rouge_greedy_meter.avg)), ('ROUGE-L/f (sampled)', '{:.4f}'.format(rouge_sampled_meter.avg)) ])) val_loss = loss_meter.avg t.write( desc + ' | valid loss {:2.2f} | valid ppl {:3.2f} | ROUGE-L (greedy): {:.4f} | ROUGE-L (sampled): {:.4f}' .format(val_loss, math.pow(2, val_loss), rouge_greedy_meter.avg, rouge_sampled_meter.avg)) # update and return the learning rate return val_loss
def train(args, epoch, batch_offset, trainer, dataset, num_gpus): """Train the model for one epoch.""" itr = dataset.dataloader( args.train_subset, num_workers=args.workers, max_tokens=args.max_tokens, seed=args.seed, epoch=epoch, max_positions=args.max_positions, sample_without_replacement=args.sample_without_replacement, skip_invalid_size_inputs_valid_test=args. skip_invalid_size_inputs_valid_test) loss_meter = AverageMeter() bsz_meter = AverageMeter() # sentences per batch wpb_meter = AverageMeter() # words per batch wps_meter = TimeMeter() # words per second clip_meter = AverageMeter() # % of updates clipped extra_meters = collections.defaultdict(lambda: AverageMeter()) desc = '| epoch {:03d}'.format(epoch) trainer.set_seed(args.seed + epoch) lr = trainer.get_lr() with progress_bar(itr, desc, leave=False) as t: for i, sample in data.skip_group_enumerator(t, num_gpus, batch_offset): loss_dict = trainer.train_step(sample) loss = loss_dict['loss'] del loss_dict[ 'loss'] # don't include in extra_meters or extra_postfix ntokens = sum(s['ntokens'] for s in sample) src_size = sum(s['src_tokens'].size(0) for s in sample) loss_meter.update(loss, ntokens) bsz_meter.update(src_size) wpb_meter.update(ntokens) wps_meter.update(ntokens) clip_meter.update(1 if loss_dict['gnorm'] > args.clip_norm else 0) extra_postfix = [] for k, v in loss_dict.items(): extra_meters[k].update(v) extra_postfix.append((k, '{:.4f}'.format(extra_meters[k].avg))) t.set_postfix(collections.OrderedDict([ ('loss', '{:.2f} ({:.2f})'.format(loss, loss_meter.avg)), ('wps', '{:5d}'.format(round(wps_meter.avg))), ('wpb', '{:5d}'.format(round(wpb_meter.avg))), ('bsz', '{:5d}'.format(round(bsz_meter.avg))), ('lr', lr), ('clip', '{:3.0f}%'.format(clip_meter.avg * 100)), ] + extra_postfix), refresh=False) if i == 0: # ignore the first mini-batch in words-per-second calculation wps_meter.reset() if args.save_interval > 0 and (i + 1) % args.save_interval == 0: save_checkpoint(trainer, args, epoch, i + 1) fmt = desc + ' | train loss {:2.2f} | train ppl {:3.2f}'.format( loss_meter.avg, get_perplexity(loss_meter.avg)) fmt += ' | s/checkpoint {:7d} | words/s {:6d} | words/batch {:6d}'.format( round(wps_meter.elapsed_time), round(wps_meter.avg), round(wpb_meter.avg)) fmt += ' | bsz {:5d} | lr {:0.6f} | clip {:3.0f}%'.format( round(bsz_meter.avg), lr, clip_meter.avg * 100) fmt += ''.join(' | {} {:.4f}'.format(k, meter.avg) for k, meter in extra_meters.items()) t.write(fmt)
def train(args, epoch, batch_offset, trainer, dataset, max_positions, num_gpus): """Train the model for one epoch.""" seed = args.seed + epoch torch.manual_seed(seed) trainer.set_seed(seed) itr = dataset.train_dataloader( args.train_subset, num_workers=args.workers, max_tokens=args.max_tokens, max_sentences=args.max_sentences, max_positions=max_positions, seed=seed, epoch=epoch, sample_without_replacement=args.sample_without_replacement, sort_by_source_size=(epoch <= args.curriculum)) loss_meter = AverageMeter() bsz_meter = AverageMeter() # sentences per batch wpb_meter = AverageMeter() # words per batch wps_meter = TimeMeter() # words per second clip_meter = AverageMeter() # % of updates clipped extra_meters = collections.defaultdict(lambda: AverageMeter()) lr = trainer.get_lr() with utils.build_progress_bar(args, itr, epoch) as t: for i, sample in data.skip_group_enumerator(t, num_gpus, batch_offset): loss_dict = trainer.train_step(sample) loss = loss_dict['loss'] del loss_dict[ 'loss'] # don't include in extra_meters or extra_postfix ntokens = sum(s['ntokens'] for s in sample) nsentences = sum(s['src_tokens'].size(0) for s in sample) loss_meter.update(loss, nsentences if args.sentence_avg else ntokens) bsz_meter.update(nsentences) wpb_meter.update(ntokens) wps_meter.update(ntokens) clip_meter.update(1 if loss_dict['gnorm'] > args.clip_norm else 0) extra_postfix = [] for k, v in loss_dict.items(): extra_meters[k].update(v) extra_postfix.append((k, extra_meters[k].avg)) t.log( collections.OrderedDict([ ('loss', loss_meter), ('wps', round(wps_meter.avg)), ('wpb', round(wpb_meter.avg)), ('bsz', round(bsz_meter.avg)), ('lr', lr), ('clip', '{:.0%}'.format(clip_meter.avg)), ] + extra_postfix)) if i == 0: # ignore the first mini-batch in words-per-second calculation wps_meter.reset() if args.save_interval > 0 and (i + 1) % args.save_interval == 0: save_checkpoint(trainer, args, epoch, i + 1) t.print( collections.OrderedDict([ ('train loss', round(loss_meter.avg, 2)), ('train ppl', get_perplexity(loss_meter.avg)), ('s/checkpoint', round(wps_meter.elapsed_time)), ('words/s', round(wps_meter.avg)), ('words/batch', round(wpb_meter.avg)), ('bsz', round(bsz_meter.avg)), ('lr', lr), ('clip', '{:3.0f}%'.format(clip_meter.avg * 100)), ] + [(k, meter.avg) for k, meter in extra_meters.items()]))
def train(args, epoch, batch_offset, trainer, dataset, max_positions, num_gpus): """Train the model for one epoch.""" seed = args.seed + epoch torch.manual_seed(seed) trainer.set_seed(seed) itr = dataset.train_dataloader( args.train_subset, num_workers=args.workers, max_tokens=args.max_tokens, max_sentences=args.max_sentences, max_positions=max_positions, seed=seed, epoch=epoch, sample_without_replacement=args.sample_without_replacement, sort_by_source_size=(epoch <= args.curriculum)) loss_meter = AverageMeter() bsz_meter = AverageMeter() # sentences per batch wpb_meter = AverageMeter() # words per batch wps_meter = TimeMeter() # words per second clip_meter = AverageMeter() # % of updates clipped extra_meters = collections.defaultdict(lambda: AverageMeter()) lr = trainer.get_lr() with utils.build_progress_bar(args, itr, epoch) as t: for i, sample in data.skip_group_enumerator(t, num_gpus, batch_offset): loss_dict = trainer.train_step(sample) loss = loss_dict['loss'] del loss_dict['loss'] # don't include in extra_meters or extra_postfix ntokens = sum(s['ntokens'] for s in sample) nsentences = sum(s['src_tokens'].size(0) for s in sample) loss_meter.update(loss, nsentences if args.sentence_avg else ntokens) bsz_meter.update(nsentences) wpb_meter.update(ntokens) wps_meter.update(ntokens) clip_meter.update(1 if loss_dict['gnorm'] > args.clip_norm else 0) extra_postfix = [] for k, v in loss_dict.items(): extra_meters[k].update(v) extra_postfix.append((k, extra_meters[k].avg)) t.log(collections.OrderedDict([ ('loss', loss_meter), ('wps', round(wps_meter.avg)), ('wpb', round(wpb_meter.avg)), ('bsz', round(bsz_meter.avg)), ('lr', lr), ('clip', '{:.0%}'.format(clip_meter.avg)), ] + extra_postfix)) if i == 0: # ignore the first mini-batch in words-per-second calculation wps_meter.reset() if args.save_interval > 0 and (i + 1) % args.save_interval == 0: save_checkpoint(trainer, args, epoch, i + 1) t.print(collections.OrderedDict([ ('train loss', round(loss_meter.avg, 2)), ('train ppl', get_perplexity(loss_meter.avg)), ('s/checkpoint', round(wps_meter.elapsed_time)), ('words/s', round(wps_meter.avg)), ('words/batch', round(wpb_meter.avg)), ('bsz', round(bsz_meter.avg)), ('lr', lr), ('clip', '{:3.0f}%'.format(clip_meter.avg * 100)), ] + [ (k, meter.avg) for k, meter in extra_meters.items() ]))
def train(args, epoch, batch_offset, trainer, criterion, dataset, num_gpus): """Train the model for one epoch.""" itr = dataset.dataloader( args.train_subset, num_workers=args.workers, max_tokens=args.max_tokens, seed=args.seed, epoch=epoch, max_positions=args.max_positions, sample_without_replacement=args.sample_without_replacement) ###print("itr:"+str(itr)) loss_meter = AverageMeter() bsz_meter = AverageMeter() # sentences per batch wpb_meter = AverageMeter() # words per batch wps_meter = TimeMeter() # words per second clip_meter = AverageMeter() # % of updates clipped gnorm_meter = AverageMeter() # gradient norm desc = '| epoch {:03d}'.format(epoch) lr = trainer.get_lr() with progress_bar(itr, desc, leave=False) as t: for i, sample in data.skip_group_enumerator(t, num_gpus, batch_offset): ###print("i:"+str(i)+" sample:"+str(sample)) ###id,src_tokens,input_tokens,input_positions,target,src_positions,ntokens ###print("i:"+str(i)+" sample len:"+str(len(sample))+" sample id:"+str(sample[0]['id'])+" sample src_tokens:"+str(sample[0]['src_tokens'][0])) aggregate_res = trainer.train_step(sample, criterion) mixed_loss = aggregate_res.loss ml_loss = aggregate_res.ml_loss grad_norm = aggregate_res.grad_norm mixed_loss = aggregate_res.loss rl_loss = aggregate_res.rl_loss mean_rouge_greedy = aggregate_res.mean_rouge_greedy mean_rouge_sampled = aggregate_res.mean_rouge_sampled mean_sum_log_prob = aggregate_res.mean_sum_log_prob ntokens = sum(s['ntokens'] for s in sample) src_size = sum(s['src_tokens'].size(0) for s in sample) loss_meter.update(ml_loss, ntokens) bsz_meter.update(src_size) wpb_meter.update(ntokens) wps_meter.update(ntokens) clip_meter.update(1 if grad_norm > args.clip_norm else 0) gnorm_meter.update(grad_norm) t.set_postfix( collections.OrderedDict([ ('loss', '{:.2f} ({:.2f})'.format(ml_loss, loss_meter.avg)), ('wps', '{:5d}'.format(round(wps_meter.avg))), ('wpb', '{:5d}'.format(round(wpb_meter.avg))), ('bsz', '{:5d}'.format(round(bsz_meter.avg))), ('lr', lr), ('clip', '{:3.0f}%'.format(clip_meter.avg * 100)), ('gnorm', '{:.4f}'.format(gnorm_meter.avg)), ])) if args.enable_rl: fmt_other = 'mixed_loss: {:^10.4f} | ml_loss: {:^10.4f}' fmt_other += '| rl_loss: {:^10.4f} | mean_rouge_greedy: {:^10.4f}' fmt_other += '| mean_rouge_sampled: {:^10.4f} | mean_sum_log_prob: {:^10.4f}' print( fmt_other.format(mixed_loss, ml_loss, rl_loss, mean_rouge_greedy, mean_rouge_sampled, mean_sum_log_prob)) if i == 0: # ignore the first mini-batch in words-per-second calculation wps_meter.reset() if args.save_interval > 0 and (i + 1) % args.save_interval == 0: trainer.save_checkpoint(args, epoch, i + 1) fmt = desc + ' | train loss {:2.2f} | train ppl {:3.2f}' fmt += ' | s/checkpoint {:7d} | words/s {:6d} | words/batch {:6d}' fmt += ' | bsz {:5d} | lr {:0.6f} | clip {:3.0f}% | gnorm {:.4f}' t.write( fmt.format(loss_meter.avg, math.pow(2, loss_meter.avg), round(wps_meter.elapsed_time), round(wps_meter.avg), round(wpb_meter.avg), round(bsz_meter.avg), lr, clip_meter.avg * 100, gnorm_meter.avg))