def print_eval(prepare_data_fun, out_label):
model_file = os.path.join(snapshot_dir, "best_model.pth")
pkl_res_file = os.path.join(snapshot_dir,
"best_model_predict_%s.pkl" % out_label)
out_file = os.path.join(snapshot_dir,
"best_model_predict_%s.json" % out_label)
data_set_test = prepare_data_fun(**cfg["data"],
**cfg["model"],
verbose=True)
data_reader_test = DataLoader(
data_set_test,
shuffle=False,
batch_size=cfg.data.batch_size,
num_workers=cfg.data.num_workers,
)
ans_dic = data_set_test.answer_dict
model = build_model(cfg, data_set_test)
model.load_state_dict(torch.load(model_file)["state_dict"])
model.eval()
question_ids, soft_max_result = run_model(model, data_reader_test,
ans_dic.UNK_idx)
print_result(
question_ids,
soft_max_result,
ans_dic,
out_file,
json_only=False,
pkl_res_file=pkl_res_file,
)
def print_eval(prepare_data_fun, out_label):
model_file = os.path.join(snapshot_dir, "best_model.pth")
pkl_res_file = os.path.join(snapshot_dir,
"best_model_predict_%s.pkl" % out_label)
out_file = os.path.join(snapshot_dir,
"best_model_predict_%s.json" % out_label)
data_set_test = prepare_data_fun(**cfg['data'],
**cfg['model'],
verbose=True)
data_reader_test = DataLoader(data_set_test,
shuffle=False,
batch_size=cfg.data.batch_size,
num_workers=cfg.data.num_workers)
ans_dic = data_set_test.answer_dict
model = build_model(cfg, data_set_test)
model.load_state_dict(torch.load(model_file)['state_dict'])
model.eval()
question_ids, soft_max_result = run_model(model,
data_reader_test,
ans_dic.UNK_idx)
print_result(question_ids,
soft_max_result,
ans_dic,
out_file,
json_only=False,
pkl_res_file=pkl_res_file)
data_reader_test = DataLoader(
data_set_test,
shuffle=False,
batch_size=batch_size,
num_workers=args.num_workers,
)
ans_dic = data_set_test.answer_dict
my_model = build_model(cfg, data_set_test)
sd = torch.load(model_file)["state_dict"]
if list(sd.keys())[0].startswith("module") and not hasattr(
my_model, "module"):
sd = multi_gpu_state_to_single(sd)
my_model.load_state_dict(sd)
my_model.eval()
print("BEGIN TESTING")
question_ids, soft_max_result = run_model(my_model, data_reader_test,
ans_dic.UNK_idx)
pkl_res_file = args.out_prefix + ".pkl" if not args.json_only else None
print_result(question_ids, soft_max_result, ans_dic, out_file,
args.json_only, pkl_res_file)
print("DONE")
args = parse_args()
result_dirs = args.res_dirs
out_file = args.out
question_ids = None
soft_max_result = None
ans_dic = None
cnt = 0
for res_dir in result_dirs:
for file in glob.glob(res_dir + "/**/*.pkl", recursive=True):
with open(file, "rb") as f:
cnt += 1
sm = pickle.load(f)
if soft_max_result is None:
soft_max_result = sm
question_ids = pickle.load(f)
ans_dic = pickle.load(f)
else:
soft_max_result += sm
print("ensemble total %d models" % cnt)
predicted_answers = np.argmax(soft_max_result, axis=1)
pkl_file = out_file + ".pkl"
print_result(question_ids, soft_max_result, ans_dic, out_file, False,
pkl_file)
print("Done")