def load_model_file(model_file): model_stuff = data.load_checkpoint(model_file) opt = utils.convert_nested_dict_to_DD(model_stuff["opt"]) state_dict = model_stuff["state_dict"] vocab = model_stuff['vocab'] return opt, state_dict, vocab
args = parser.parse_args() split = args.split # Generate configuration files depending on experiment being run utils.generate_config_files("atomic", args.experiment_num, eval_mode=True) # Loads the correct configuration file config_file = "config/atomic/config_{}.json".format(args.experiment_num) # Read config file to option config = cfg.read_config(cfg.load_config(config_file)) cfg.device = config.gpu_index eval_opt = cfg.get_eval_parameters(config) model_stuff = data.load_checkpoint(args.model_name) opt = model_stuff["opt"] opt.eval.update(eval_opt) opt.train.dynamic.epoch = 0 print("Loading Data") categories = opt.data.categories path = "data/atomic/processed/generation/{}.pickle".format( utils.make_name_string(opt.data)) data_loader = data.make_data_loader(opt, categories) loaded = data_loader.load_data(path)
# eval_mode = True means changes are taken from config/atomic/eval_changes.json utils.generate_config_files("atomic", args.experiment_num, eval_mode=True) # Loads the correct configuration file config_file = "config/atomic/config_{}.json".format(args.experiment_num) print(config_file) # Read config file to option config = cfg.read_config(cfg.load_config(config_file)) cfg.device = config.gpu_index eval_opt = cfg.get_eval_parameters(config) # Batch multiple models model_file = data.load_checkpoint(args.model_name) opt = model_file["opt"] opt.eval.update(eval_opt) print("Loading Data") # Do multiple sets of categories: # compute individual perplexity of categories in addition to total perplexity if len(opt.data.categories) == 1: set_of_categories = [opt.data.categories] else: set_of_categories = [opt.data.categories] + [[i] for i in opt.data.categories] print(set_of_categories)
def load_model_file(model_file): model_stuff = data.load_checkpoint(model_file) opt = model_stuff["opt"] state_dict = model_stuff["state_dict"] return opt, state_dict