def benchmark_features(): i_err = {thr: 0 for thr in rng} v_err = {thr: 0 for thr in rng} model = cv2.xfeatures2d.SURF_create() with torch.no_grad(): data_loader = get_dataloader("view") for i_batch, sample_batched in tqdm(enumerate(data_loader, 1)): batch = parse_batch(sample_batched, device) for thr in rng: TPNN, PNN, TPNNT, PNNT, TPNNDR, PNNDR = eval_sift( model, batch, 1, thr) PreNN = TPNN / PNN PreNNT = TPNNT / PNNT PreNNDR = TPNNDR / PNNDR meanms = (PreNN + PreNNT + PreNNDR) / 3 v_err[thr] += meanms n_v = i_batch with torch.no_grad(): data_loader = get_dataloader("illu") for i_batch, sample_batched in tqdm(enumerate(data_loader, 1)): batch = parse_batch(sample_batched, device) for thr in rng: TPNN, PNN, TPNNT, PNNT, TPNNDR, PNNDR = eval_sift( model, batch, 1, thr) PreNN = TPNN / PNN PreNNT = TPNNT / PNNT PreNNDR = TPNNDR / PNNDR meanms = (PreNN + PreNNT + PreNNDR) / 3 i_err[thr] += meanms n_i = i_batch return i_err, v_err, n_i, n_v
def evaluate(data_source): model.eval() PreNN, PreNNT, PreNNDR = 0, 0, 0 with torch.no_grad(): for i_batch, sample_batched in enumerate(data_source, 1): batch = parse_batch(sample_batched, device) TPNN, PNN, TPNNT, PNNT, TPNNDR, PNNDR = eval_model( model.module if mgpu else model, batch, cfg.MODEL.DES_THRSH, cfg.MODEL.COO_THRSH, ) PreNN += TPNN / PNN PreNNT += TPNNT / PNNT PreNNDR += TPNNDR / PNNDR length = len(data_source) PreNN, PreNNT, PreNNDR = (PreNN / length, PreNNT / length, PreNNDR / length) meanms = (PreNN + PreNNT + PreNNDR) / 3 checkpoint_name = ( f"NN_{PreNN:.3f}_NNT_{PreNNT:.3f}_NNDR_{PreNNDR:.3f}_MeanMS_{meanms:.3f}" ) return checkpoint_name, meanms
def benchmark_features(): i_err = {thr: 0 for thr in rng} v_err = {thr: 0 for thr in rng} # Creating CNN model model = D2Net(model_file="/home/wang/d2-net/models/d2_tf.pth", use_relu=True, use_cuda=use_cuda) with torch.no_grad(): data_loader = get_dataloader("view") for i_batch, sample_batched in tqdm(enumerate(data_loader, 1)): batch = parse_batch(sample_batched, device) for thr in rng: TPNN, PNN, TPNNT, PNNT, TPNNDR, PNNDR = eval_d2net( model, batch, 1, thr) PreNN = TPNN / PNN PreNNT = TPNNT / PNNT PreNNDR = TPNNDR / PNNDR meanms = (PreNN + PreNNT + PreNNDR) / 3 v_err[thr] += meanms n_v = i_batch with torch.no_grad(): data_loader = get_dataloader("illu") for i_batch, sample_batched in tqdm(enumerate(data_loader, 1)): batch = parse_batch(sample_batched, device) for thr in rng: TPNN, PNN, TPNNT, PNNT, TPNNDR, PNNDR = eval_d2net( model, batch, 1, thr) PreNN = TPNN / PNN PreNNT = TPNNT / PNNT PreNNDR = TPNNDR / PNNDR meanms = (PreNN + PreNNT + PreNNDR) / 3 i_err[thr] += meanms n_i = i_batch return i_err, v_err, n_i, n_v
Grayscale(), Normalize(mean=mean, std=std), Rescale((960, 1280)), Rescale((480, 640)), ToTensor() ]), ), batch_size=1, shuffle=False, num_workers=0) useful_list = [] repeat_list = [] with torch.no_grad(): for i_batch, sample_batched in enumerate(data_loader, 1): im1_data, im1_info, homo12, im2_data, im2_info, homo21, im1_raw, im2_raw = parse_batch( sample_batched, device) # (angle, class_id, octave, pt, response, size) keypoints1, scores1, descriptors1 = process_multiscale( im1_data.repeat(1, 3, 1, 1), model, scales=[1]) keypoints2, scores2, descriptors2 = process_multiscale( im2_data.repeat(1, 3, 1, 1), model, scales=[1]) kp1c, _ = topk(keypoints1, scores1, args.k) kp2c, _ = topk(keypoints2, scores2, args.k) im1_data, im1_info, homo12, im2_data, im2_info, homo21, im1_raw, im2_raw = parse_batch_np( sample_batched, mean, std) repeatable, useful = caluseful(kp1c, kp2c, homo12, im2_data) useful_list.append(useful), repeat_list.append(repeatable) usefuls = np.array(useful_list)
def train(): start_time = time.time() for i_batch, sample_batched in enumerate(train_data, 1): model.train() batch = parse_batch(sample_batched, device) with autograd.detect_anomaly(): for des_train in range(0, cfg.TRAIN.DES): model.zero_grad() des_optim.zero_grad() endpoint = model(batch) _, _, desloss = (model.module.criterion(endpoint) if mgpu else model.criterion(endpoint)) desloss.backward() des_optim.step() for det_train in range(0, cfg.TRAIN.DET): model.zero_grad() det_optim.zero_grad() endpoint = model(batch) _, detloss, _ = (model.module.criterion(endpoint) if mgpu else model.criterion(endpoint)) detloss.backward() det_optim.step() Lr_Schechuler(cfg.TRAIN.DET_LR_SCHEDULE, det_optim, epoch, cfg) Lr_Schechuler(cfg.TRAIN.DES_LR_SCHEDULE, des_optim, epoch, cfg) # log if i_batch % cfg.TRAIN.LOG_INTERVAL == 0 and i_batch > 0: elapsed = time.time() - start_time model.eval() with torch.no_grad(): eptr = model(parse_unsqueeze(train_data.dataset[0], device)) PLT, cur_detloss, cur_desloss = ( model.module.criterion(eptr) if mgpu else model.criterion(eptr)) PLTS = PLT["scalar"] PLTS["Accuracy"] = getAC(eptr["im1_lpdes"], eptr["im1_rpdes"]) PLTS["det_lr"] = det_optim.param_groups[0]["lr"] PLTS["des_lr"] = des_optim.param_groups[0]["lr"] if mgpu: mgpu_merge(PLTS) iteration = (epoch - 1) * len(train_data) + (i_batch - 1) writer_log(train_writer, PLT["scalar"], iteration) pstring = ( "epoch {:2d} | {:4d}/{:4d} batches | ms {:4.02f} | " "sco {:07.05f} | pair {:05.03f} | des {:05.03f} |". format( epoch, i_batch, len(train_data) // cfg.TRAIN.BATCH_SIZE, elapsed / cfg.TRAIN.LOG_INTERVAL, PLTS["score_loss"], PLTS["pair_loss"], PLTS["hard_loss"], )) # eval log # parsed_valbatch = parse_unsqueeze(val_data.dataset[0], device) # ept = model(parsed_valbatch) ept = model(parse_unsqueeze(val_data.dataset[0], device)) PLT, _, _ = (model.module.criterion(ept) if mgpu else model.criterion(ept)) PLTS = PLT["scalar"] PLTS["Accuracy"] = getAC(ept["im1_lpdes"], ept["im1_rpdes"]) writer_log(test_writer, PLT["scalar"], iteration) print(f"{gct()} | {pstring}") start_time = time.time()
cfg.TRAIN.TOPK) print(f"{gct()} : to device") device = torch.device("cuda") model = model.to(device) resume = args.resume print(f"{gct()} : in {resume}") checkpoint = torch.load(resume) model.load_state_dict(checkpoint["state_dict"]) print(f"{gct()} : start eval") model.eval() PreNN, PreNNT, PreNNDR = 0, 0, 0 with torch.no_grad(): for i_batch, sample_batched in enumerate(data_loader, 1): batch = parse_batch(sample_batched, device) TPNN, PNN, TPNNT, PNNT, TPNNDR, PNNDR = eval_model( model, batch, cfg.MODEL.DES_THRSH, cfg.MODEL.COO_THRSH) PreNN += TPNN / PNN PreNNT += TPNNT / PNNT PreNNDR += TPNNDR / PNNDR length = len(data_loader) PreNN, PreNNT, PreNNDR = (PreNN / length, PreNNT / length, PreNNDR / length) meanms = (PreNN + PreNNT + PreNNDR) / 3 checkpoint_name = (f"data: {args.data} " f"len: {len(data_loader):3d} " f"ct: {args.ct}\t"
def train(): start_time = time.time() for i_batch, sample_batched in enumerate(train_data, 1): det.train() des.train() batch = parse_batch(sample_batched, device) with autograd.detect_anomaly(): for des_train in range(0, cfg.TRAIN.DES): des.zero_grad() des_optim.zero_grad() im1_data, im1_info, homo12, im2_data, im2_info, homo21, im1_raw, im2_raw = batch score_maps, orint_maps = det(im1_data) im1_rawsc, im1_scale, im1_orin = handle_det_out( score_maps, orint_maps, det.scale_list, det.score_com_strength, det.scale_com_strength) score_maps, orint_maps = det(im2_data) im2_rawsc, im2_scale, im2_orin = handle_det_out( score_maps, orint_maps, det.scale_list, det.score_com_strength, det.scale_com_strength) im1_gtscale, im1_gtorin = gt_scale_orin( im2_scale, im2_orin, homo12, homo21) im2_gtscale, im2_gtorin = gt_scale_orin( im1_scale, im1_orin, homo21, homo12) im2_score = filter_border(im2_rawsc) im1w_score = warp(im2_score, homo12) im1_gtsc, im1_topkmask, im1_topkvalue = det.process( im1w_score) im1_score = filter_border(im1_rawsc) im2w_score = warp(im1_score, homo21) im2_gtsc, im2_topkmask, im2_topkvalue = det.process( im2w_score) im1_ppair, im1_limc, im1_rimcw = pair( im1_topkmask, im1_topkvalue, im1_scale, im1_orin, im1_info, im1_raw, homo12, im2_gtscale, im2_gtorin, im2_info, im2_raw, cfg.PATCH.SIZE, ) im2_ppair, im2_limc, im2_rimcw = pair( im2_topkmask, im2_topkvalue, im2_scale, im2_orin, im2_info, im2_raw, homo21, im1_gtscale, im1_gtorin, im1_info, im1_raw, cfg.PATCH.SIZE, ) im1_lpatch, im1_rpatch = im1_ppair.chunk( chunks=2, dim=1) # each is (N, 32, 32) im2_lpatch, im2_rpatch = im2_ppair.chunk( chunks=2, dim=1) # each is (N, 32, 32) im1_lpatch = des.input_norm(im1_lpatch) im2_lpatch = des.input_norm(im2_lpatch) im1_rpatch = des.input_norm(im1_rpatch) im2_rpatch = des.input_norm(im2_rpatch) im1_lpdes, im1_rpdes = des(im1_lpatch), des(im1_rpatch) im2_lpdes, im2_rpdes = des(im2_lpatch), des(im2_rpatch) endpoint = { "im1_limc": im1_limc, "im1_rimcw": im1_rimcw, "im2_limc": im2_limc, "im2_rimcw": im2_rimcw, "im1_lpdes": im1_lpdes, "im1_rpdes": im1_rpdes, "im2_lpdes": im2_lpdes, "im2_rpdes": im2_rpdes, } desloss = (des.module.criterion(endpoint) if mgpu else des.criterion(endpoint)) desloss.backward() des_optim.step() for det_train in range(0, cfg.TRAIN.DET): det.zero_grad() det_optim.zero_grad() im1_data, im1_info, homo12, im2_data, im2_info, homo21, im1_raw, im2_raw = batch score_maps, orint_maps = det(im1_data) im1_rawsc, im1_scale, im1_orin = handle_det_out( score_maps, orint_maps, det.scale_list, det.score_com_strength, det.scale_com_strength) score_maps, orint_maps = det(im2_data) im2_rawsc, im2_scale, im2_orin = handle_det_out( score_maps, orint_maps, det.scale_list, det.score_com_strength, det.scale_com_strength) im2_score = filter_border(im2_rawsc) im1w_score = warp(im2_score, homo12) im1_visiblemask = warp( im2_score.new_full(im2_score.size(), fill_value=1, requires_grad=True), homo12, ) im1_gtsc, im1_topkmask, im1_topkvalue = det.process( im1w_score) im1_score = filter_border(im1_rawsc) im2w_score = warp(im1_score, homo21) im2_visiblemask = warp( im2_score.new_full(im1_score.size(), fill_value=1, requires_grad=True), homo21, ) im2_gtsc, im2_topkmask, im2_topkvalue = det.process( im2w_score) im1_score = det.process(im1_rawsc)[0] im2_score = det.process(im2_rawsc)[0] im1_predpair, _, _ = pair( im1_topkmask, im1_topkvalue, im1_scale, im1_orin, im1_info, im1_raw, homo12, im2_scale, im2_orin, im2_info, im2_raw, cfg.PATCH.SIZE, ) im2_predpair, _, _ = pair( im2_topkmask, im2_topkvalue, im2_scale, im2_orin, im2_info, im2_raw, homo21, im1_scale, im1_orin, im1_info, im1_raw, cfg.PATCH.SIZE, ) # each is (N, 32, 32) im1_lpredpatch, im1_rpredpatch = im1_predpair.chunk( chunks=2, dim=1) im2_lpredpatch, im2_rpredpatch = im2_predpair.chunk( chunks=2, dim=1) im1_lpredpatch = des.input_norm(im1_lpredpatch) im2_lpredpatch = des.input_norm(im2_lpredpatch) im1_rpredpatch = des.input_norm(im1_rpredpatch) im2_rpredpatch = des.input_norm(im2_rpredpatch) im1_lpreddes, im1_rpreddes = des(im1_lpredpatch), des( im1_rpredpatch) im2_lpreddes, im2_rpreddes = des(im2_lpredpatch), des( im2_rpredpatch) endpoint = { "im1_score": im1_score, "im1_gtsc": im1_gtsc, "im1_visible": im1_visiblemask, "im2_score": im2_score, "im2_gtsc": im2_gtsc, "im2_visible": im2_visiblemask, "im1_lpreddes": im1_lpreddes, "im1_rpreddes": im1_rpreddes, "im2_lpreddes": im2_lpreddes, "im2_rpreddes": im2_rpreddes, } detloss = (det.module.criterion(endpoint) if mgpu else det.criterion(endpoint)) detloss.backward() det_optim.step() Lr_Schechuler(cfg.TRAIN.DET_LR_SCHEDULE, det_optim, epoch, cfg) Lr_Schechuler(cfg.TRAIN.DES_LR_SCHEDULE, des_optim, epoch, cfg) # log if i_batch % cfg.TRAIN.LOG_INTERVAL == 0 and i_batch > 0: elapsed = time.time() - start_time det.eval() des.eval() with torch.no_grad(): parsed_trainbatch = parse_unsqueeze( train_data.dataset[0], device) endpoint = get_all_endpoints(det, des, parsed_trainbatch) detloss = (det.module.criterion(endpoint) if mgpu else det.criterion(endpoint)) desloss = (des.module.criterion(endpoint) if mgpu else des.criterion(endpoint)) PLT_SCALAR = {} PLT = {"scalar": PLT_SCALAR} PLT_SCALAR["pair_loss"] = detloss PLT_SCALAR["hard_loss"] = desloss PLTS = PLT["scalar"] PLTS["Accuracy"] = getAC(endpoint["im1_lpdes"], endpoint["im1_rpdes"]) PLTS["det_lr"] = det_optim.param_groups[0]["lr"] PLTS["des_lr"] = des_optim.param_groups[0]["lr"] if mgpu: mgpu_merge(PLTS) iteration = (epoch - 1) * len(train_data) + (i_batch - 1) writer_log(train_writer, PLT["scalar"], iteration) pstring = ( "epoch {:2d} | {:4d}/{:4d} batches | ms {:4.02f} | " "pair {:05.03f} | des {:05.03f} |".format( epoch, i_batch, len(train_data) // cfg.TRAIN.BATCH_SIZE, elapsed / cfg.TRAIN.LOG_INTERVAL, PLTS["pair_loss"], PLTS["hard_loss"], )) # eval log parsed_valbatch = parse_unsqueeze(val_data.dataset[0], device) ept = get_all_endpoints(det, des, parsed_valbatch) detloss = (det.module.criterion(endpoint) if mgpu else det.criterion(endpoint)) desloss = (des.module.criterion(endpoint) if mgpu else des.criterion(endpoint)) PLT_SCALAR = {} PLT = {"scalar": PLT_SCALAR} PLT_SCALAR["pair_loss"] = detloss PLT_SCALAR["hard_loss"] = desloss PLTS = PLT["scalar"] PLTS["Accuracy"] = getAC(ept["im1_lpdes"], ept["im1_rpdes"]) writer_log(test_writer, PLT["scalar"], iteration) print(f"{gct()} | {pstring}") start_time = time.time()