def drop_checkpoint(self, opt, epoch, fields, valid_stats):
""" Save a resumable checkpoint.
Args:
opt (dict): option object
epoch (int): epoch number
fields (dict): fields and vocabulary
valid_stats : statistics of last validation run
"""
real_model = (self.model.module if isinstance(
self.model, nn.DataParallel) else self.model)
real_generator = (real_model.generator.module if isinstance(
real_model.generator, nn.DataParallel) else real_model.generator)
model_state_dict = real_model.state_dict()
model_state_dict = {
k: v
for k, v in model_state_dict.items() if 'generator' not in k
}
generator_state_dict = real_generator.state_dict()
checkpoint = {
'model': model_state_dict,
'generator': generator_state_dict,
'vocab': inputters.save_fields_to_vocab(fields),
'opt': opt,
'epoch': epoch,
'optim': self.optim,
}
torch.save(
checkpoint, '%s_acc_%.2f_ppl_%.2f_e%d.pt' %
(opt.save_model, valid_stats.accuracy(), valid_stats.ppl(), epoch))
def build_save_vocab(train_dataset, fields, opt):
fields = inputters.build_vocab(
train_dataset, fields, opt.data_type, opt.share_vocab,
opt.src_vocab, opt.src_vocab_size, opt.src_words_min_frequency,
opt.tgt_vocab, opt.tgt_vocab_size, opt.tgt_words_min_frequency
)
vocab_path = opt.save_data + '.vocab.pt'
torch.save(inputters.save_fields_to_vocab(fields), vocab_path)
def build_save_vocab(train_dataset, fields, opt):
""" Building and saving the vocab """
fields = inputters.build_vocab(
train_dataset, fields, opt.data_type, opt.share_vocab, opt.src_vocab,
opt.src_vocab_size, opt.src_words_min_frequency, opt.tgt_vocab,
opt.tgt_vocab_size, opt.tgt_words_min_frequency, opt.extra_vocab)
# Can't save fields, so remove/reconstruct at training time.
vocab_file = opt.save_data + '.vocab.pt'
torch.save(inputters.save_fields_to_vocab(fields), vocab_file)
def build_save_vocab(train_dataset, data_type, fields, opt):
""" Building and saving the vocab """
fields = inputters.build_vocab(train_dataset, data_type, fields,
opt.share_vocab, opt.src_vocab_size,
opt.src_words_min_frequency,
opt.tgt_vocab_size,
opt.tgt_words_min_frequency)
# Can't save fields, so remove/reconstruct at training time.
vocab_file = opt.save_data + '.vocab.pt'
logger.info("saving vocabulary {}".format(vocab_file))
torch.save(inputters.save_fields_to_vocab(fields), vocab_file)
def build_save_vocab(train_dataset, fields, opt):
""" Building and saving the vocab """
fields = inputters.build_vocab(train_dataset, fields, "text",
opt.share_vocab, opt.src_vocab,
opt.src_vocab_size,
opt.src_words_min_frequency, opt.tgt_vocab,
opt.tgt_vocab_size,
opt.tgt_words_min_frequency)
# Can't save fields, so remove/reconstruct at training time.
vocab_file = opt.save_data + '.vocab.pt'
# torch.save(inputters.save_fields_to_vocab(fields), vocab_file)
with open(vocab_file, 'wb') as f:
pickle.dump(inputters.save_fields_to_vocab(fields), f)
def build_save_vocab(train_dataset, fields, savepath, opt):
""" Building and saving the vocab """
fields = inputters.build_vocab(train_dataset,
fields,
data_type='text',
share_vocab=True,
src_vocab_path='',
src_vocab_size=100,
src_words_min_frequency=1,
tgt_vocab_path='',
tgt_vocab_size=100,
tgt_words_min_frequency=1)
# Can't save fields, so remove/reconstruct at training time.
vocab_file = savepath + '/vocab.pt'
with open(vocab_file, 'wb') as f:
pickle.dump(inputters.save_fields_to_vocab(fields), f)
def build_save_vocab(fields, opt):
""" Building and saving the vocab """
fields = inputters.build_vocab_vec(fields)
# Can't save fields, so remove/reconstruct at training time.
vocab_file = opt.save_data + '.vocab.pt'
torch.save(inputters.save_fields_to_vocab(fields), vocab_file)