def print_eval(prepare_data_fun, out_label): model_file = os.path.join(snapshot_dir, "best_model.pth") pkl_res_file = os.path.join(snapshot_dir, "best_model_predict_%s.pkl" % out_label) out_file = os.path.join(snapshot_dir, "best_model_predict_%s.json" % out_label) data_set_test = prepare_data_fun(**cfg["data"], **cfg["model"], verbose=True) data_reader_test = DataLoader( data_set_test, shuffle=False, batch_size=cfg.data.batch_size, num_workers=cfg.data.num_workers, ) ans_dic = data_set_test.answer_dict model = build_model(cfg, data_set_test) model.load_state_dict(torch.load(model_file)["state_dict"]) model.eval() question_ids, soft_max_result = run_model(model, data_reader_test, ans_dic.UNK_idx) print_result( question_ids, soft_max_result, ans_dic, out_file, json_only=False, pkl_res_file=pkl_res_file, )
def print_eval(prepare_data_fun, out_label): model_file = os.path.join(snapshot_dir, "best_model.pth") pkl_res_file = os.path.join(snapshot_dir, "best_model_predict_%s.pkl" % out_label) out_file = os.path.join(snapshot_dir, "best_model_predict_%s.json" % out_label) data_set_test = prepare_data_fun(**cfg['data'], **cfg['model'], verbose=True) data_reader_test = DataLoader(data_set_test, shuffle=False, batch_size=cfg.data.batch_size, num_workers=cfg.data.num_workers) ans_dic = data_set_test.answer_dict model = build_model(cfg, data_set_test) model.load_state_dict(torch.load(model_file)['state_dict']) model.eval() question_ids, soft_max_result = run_model(model, data_reader_test, ans_dic.UNK_idx) print_result(question_ids, soft_max_result, ans_dic, out_file, json_only=False, pkl_res_file=pkl_res_file)
data_reader_test = DataLoader( data_set_test, shuffle=False, batch_size=batch_size, num_workers=args.num_workers, ) ans_dic = data_set_test.answer_dict my_model = build_model(cfg, data_set_test) sd = torch.load(model_file)["state_dict"] if list(sd.keys())[0].startswith("module") and not hasattr( my_model, "module"): sd = multi_gpu_state_to_single(sd) my_model.load_state_dict(sd) my_model.eval() print("BEGIN TESTING") question_ids, soft_max_result = run_model(my_model, data_reader_test, ans_dic.UNK_idx) pkl_res_file = args.out_prefix + ".pkl" if not args.json_only else None print_result(question_ids, soft_max_result, ans_dic, out_file, args.json_only, pkl_res_file) print("DONE")
args = parse_args() result_dirs = args.res_dirs out_file = args.out question_ids = None soft_max_result = None ans_dic = None cnt = 0 for res_dir in result_dirs: for file in glob.glob(res_dir + "/**/*.pkl", recursive=True): with open(file, "rb") as f: cnt += 1 sm = pickle.load(f) if soft_max_result is None: soft_max_result = sm question_ids = pickle.load(f) ans_dic = pickle.load(f) else: soft_max_result += sm print("ensemble total %d models" % cnt) predicted_answers = np.argmax(soft_max_result, axis=1) pkl_file = out_file + ".pkl" print_result(question_ids, soft_max_result, ans_dic, out_file, False, pkl_file) print("Done")