def drop_checkpoint(self, opt, epoch, valid_stats, model_opt=None):
""" Save a resumable checkpoint.
Args:
opt (dict): option object
epoch (int): epoch number
fields (dict): fields and vocabulary
valid_stats : statistics of last validation run
"""
real_model = (self.model.module
if isinstance(self.model, nn.DataParallel)
else self.model)
model_state_dict = real_model.state_dict()
model_state_dict = {k: v for k, v in model_state_dict.items()
if 'generator' not in k}
checkpoint = {
'model': model_state_dict,
'opt': opt if not model_opt else model_opt,
'epoch': epoch,
'optim': self.optim,
}
# Add critic in checkpoint
if hasattr(self, 'critic'):
checkpoint['critic'] = self.critic.state_dict()
path = self.checkpoint_path(epoch, opt, valid_stats)
create_path(path)
if not opt.best_only:
print('Save checkpoint {path}'.format(path=path))
torch.save(checkpoint, path)
self.save_best_checkpoint(checkpoint, opt, valid_stats)
return path
def drop_checkpoint(self, opt, epoch, valid_stats, model_opt=None):
""" Save a resumable checkpoint.
Args:
opt (dict): option object
epoch (int): epoch number
fields (dict): fields and vocabulary
valid_stats : statistics of last validation run
"""
real_model = (self.model.module if isinstance(
self.model, nn.DataParallel) else self.model)
real_generator = (real_model.generator.module if isinstance(
real_model.generator, nn.DataParallel) else real_model.generator)
model_state_dict = real_model.state_dict()
model_state_dict = {
k: v
for k, v in model_state_dict.items() if 'generator' not in k
}
generator_state_dict = real_generator.state_dict()
checkpoint = {
'model': model_state_dict,
'generator': generator_state_dict,
'opt': opt if not model_opt else model_opt,
'epoch': epoch,
'optim': self.optim,
}
path = self.checkpoint_path(epoch, opt, valid_stats)
create_path(path)
print 'Save checkpoint {path}'.format(path=path)
torch.save(checkpoint, path)
self.save_best_checkpoint(checkpoint, opt, valid_stats)
loading_timer = tm.time()
schema = Schema(model_args.schema_path, None)
data_generator = get_data_generator(args, model_args, schema)
mappings = data_generator.mappings
if args.vocab_only:
import sys; sys.exit()
if args.verbose:
print("Finished loading and pre-processing data, took {:.1f} seconds".format(tm.time() - loading_timer))
# TODO: load from checkpoint
ckpt = None
# Build the model
model = build_model(model_args, args, mappings, ckpt, model_path=args.agent_checkpoint)
tally_parameters(model)
create_path(args.model_path)
config_path = os.path.join(args.model_path, 'config.json')
write_json(vars(args), config_path)
builder = UtteranceBuilder(mappings['tgt_vocab'], 1, has_tgt=True)
# Build optimizer and trainer
optim = build_optim(args, model, ckpt)
# vocab is used to make_loss, so use target vocab
trainer = build_trainer(args, model, mappings['tgt_vocab'], optim)
trainer.builder = builder
# Perform actual training
trainer.learn(args, data_generator, report_func)
help='JSON file to save evaluation results')
parser.add_argument('--best',
default=False,
action='store_true',
help='Test using the best model on dev set')
parser.add_argument('--verbose',
default=False,
action='store_true',
help='More prints')
options.add_data_generator_arguments(parser)
options.add_model_arguments(parser)
cocoa.options.add_trainer_arguments(parser)
args = parser.parse_args()
random.seed(args.random_seed)
create_path(args.stats_file)
logstats.init(args.stats_file, args.verbose)
logstats.add_args('config', args)
model_args = args
if torch.cuda.is_available() and not args.gpuid:
print("WARNING: You have a CUDA device, should run with -gpuid 0")
if args.gpuid:
cuda.set_device(args.gpuid[0])
if args.random_seed > 0:
torch.cuda.manual_seed(args.random_seed)
loading_timer = tm.time()
schema = Schema(model_args.schema_path, None)