def eval_single_ckpt(model,
test_loader,
args,
eval_output_dir,
logger,
epoch_id,
dist_test=False):
# load checkpoint
# print("HERE BITCH", args.ckpt)
# args.ckpt = '/home/mrsd2/Documents/vlr-project/Voxel-R-CNN/output/voxel_rcnn/voxel_rcnn_pedestrian/default/ckpt/checkpoint_epoch_80.pth'
# cwd = os.getcwd()
# print("HERE BITCH", cwd)
model.load_params_from_file(filename=args.ckpt,
logger=logger,
to_cpu=dist_test)
model.cuda()
# start evaluation
# print("HERE YE FOOLS", args.save_to_file)
# print("HERE YE FOOLS HARKEN", eval_output_dir)
eval_utils.eval_one_epoch(cfg,
model,
test_loader,
epoch_id,
logger,
dist_test=dist_test,
result_dir=eval_output_dir,
save_to_file=args.save_to_file)
def eval_single_ckpt(model, test_loader, args, eval_output_dir, logger, epoch_id):
# load checkpoint
model.load_params_from_file(filename=args.ckpt, logger=logger)
model.cuda()
# start evaluation
eval_utils.eval_one_epoch(
model, test_loader, epoch_id, logger, result_dir=eval_output_dir, save_to_file=args.save_to_file
)
def eval_single_ckpt(model, test_loader, args, eval_output_dir, logger, epoch_id, dist_test=False, state_name='model_state'):
# load checkpoint
model.load_params_from_file(filename=args.ckpt, logger=logger, to_cpu=dist_test, state_name=state_name)
model.cuda()
# start evaluation
eval_utils.eval_one_epoch(
cfg, model, test_loader, epoch_id, logger, dist_test=dist_test,
result_dir=eval_output_dir, save_to_file=args.save_to_file, args=args
)
def eval_single_ckpt(model, test_loader, args, eval_output_dir, logger, epoch_id, dist_test=False):
# load checkpoint
model.load_params_from_file(filename=args.ckpt, logger=logger, to_cpu=dist_test)
model.cuda()
# start evaluation
if cfg.DATA_CONFIG.IMAGE_SEG_TASK:
eval_utils.eval_one_epoch_for_semantic(
cfg, model, test_loader, epoch_id, logger, dist_test=dist_test,
result_dir=eval_output_dir, save_to_file=args.save_to_file
)
else:
eval_utils.eval_one_epoch(
cfg, model, test_loader, epoch_id, logger, dist_test=dist_test,
result_dir=eval_output_dir, save_to_file=args.save_to_file
)
def repeat_eval_ckpt(model, test_loader, args, eval_output_dir, logger,
ckpt_dir):
# evaluated ckpt record
ckpt_record_file = eval_output_dir / ('eval_list_%s.txt' %
cfg.MODEL.TEST.SPLIT)
with open(ckpt_record_file, 'a'):
pass
# tensorboard log
tb_log = SummaryWriter(
log_dir=str(eval_output_dir /
('tensorboard_%s' % cfg.MODEL.TEST.SPLIT)))
total_time = 0
first_eval = True
while True:
# check whether there is checkpoint which is not evaluated
cur_epoch_id, cur_ckpt = get_no_evaluated_ckpt(ckpt_dir,
ckpt_record_file, args)
if cur_epoch_id == -1 or int(float(cur_epoch_id)) < args.start_epoch:
wait_second = 30
print(
'Wait %s seconds for next check (progress: %.1f / %d minutes): %s \r'
% (wait_second, total_time * 1.0 / 60, args.max_waiting_mins,
ckpt_dir),
end='',
flush=True)
time.sleep(wait_second)
total_time += 30
if total_time > args.max_waiting_mins * 60 and (first_eval is
False):
break
continue
total_time = 0
first_eval = False
model.load_params_from_file(filename=cur_ckpt, logger=logger)
model.cuda()
# start evaluation
cur_result_dir = eval_output_dir / (
'epoch_%s' % cur_epoch_id) / cfg.MODEL.TEST.SPLIT
tb_dict = eval_utils.eval_one_epoch(model,
test_loader,
cur_epoch_id,
logger,
result_dir=cur_result_dir,
save_to_file=args.save_to_file)
for key, val in tb_dict.items():
tb_log.add_scalar(key, val, cur_epoch_id)
# record this epoch which has been evaluated
with open(ckpt_record_file, 'a') as f:
print('%s' % cur_epoch_id, file=f)
logger.info('Epoch %s has been evaluated' % cur_epoch_id)
