def execute_experiment(cfg):
# -- set seed --
np.random.seed(seed)
torch.manual_seed(seed)
# -- load dataset --
data,loaders = load_image_dataset(cfg)
image_iter = iter(loaders.tr)
# -- run over images --
for i_index in range(NUM_BATCHES):
# -- sample & unpack batch --
sample = next(image_iter)
sample_to_cuda(sample)
dyn_noisy = sample['noisy'] # dynamics and noise
dyn_clean = sample['burst'] # dynamics and no noise
static_noisy = sample['snoisy'] # no dynamics and noise
static_clean = sample['sburst'] # no dynamics and no noise
flow_gt = sample['flow']
image_index = sample['index']
tl_index = sample['tl_index']
rng_state = sample['rng_state']
if cfg.noise_params.ntype == "pn":
dyn_noisy = anscombe.forward(dyn_noisy)
def main():
seed = 234
np.random.seed(seed)
torch.manual_seed(seed)
# -- settings --
cfg = get_cfg_defaults()
cfg.use_anscombe = True
cfg.noise_params.ntype = 'g'
cfg.noise_params.g.std = 25.
cfg.nframes = 3
cfg.num_workers = 0
cfg.dynamic_info.nframes = cfg.nframes
cfg.dynamic_info.ppf = 10
cfg.nblocks = 3
cfg.patchsize = 10
cfg.gpuid = 1
cfg.device = f"cuda:{cfg.gpuid}"
# -- load dataset --
data, loaders = load_image_dataset(cfg)
train_iter = iter(loaders.tr)
# -- fetch sample --
sample = next(train_iter)
sample_to_cuda(sample)
# -- unpack data --
noisy, clean = sample['noisy'], sample['burst']
# -- save ave image --
save_image(torch.mean(noisy[:, 0], dim=0),
SAVE_DIR / "./bootstrap_noisy_ave.png")
# -- format for plots --
print("noisy.shape", noisy.shape)
noisy = rearrange(noisy[:, 0], 't c h w -> t h w c')
clean = rearrange(clean[:, 0], 't c h w -> t h w c')
plot_bootstrapping(noisy, clean)
def test_nnf():
# -- get config --
cfg = config()
# -- set random seed --
set_seed(cfg.random_seed)
# -- load dataset --
data, loaders = load_image_dataset(cfg)
image_iter = iter(loaders.tr)
# -- get score function --
score_fxn = get_score_function(cfg.score_fxn_name)
# -- some constants --
NUM_BATCHES = 2
nframes, nblocks = cfg.nframes, cfg.nblocks
patchsize = cfg.patchsize
check_parameters(nblocks, patchsize)
# -- create evaluator
iterations, K = 10, 2
subsizes = [2, 2, 2, 2, 2]
evaluator = combo.eval_scores.EvalBlockScores(score_fxn, patchsize, 100,
None)
# -- iterate over images --
for image_bindex in range(NUM_BATCHES):
# -- sample & unpack batch --
sample = next(image_iter)
sample_to_cuda(sample)
dyn_noisy = sample['noisy'] # dynamics and noise
dyn_clean = sample['burst'] # dynamics and no noise
static_noisy = sample['snoisy'] # no dynamics and noise
static_clean = sample['sburst'] # no dynamics and no noise
flow_gt = sample['flow']
# -- shape info --
T, B, C, H, W = dyn_noisy.shape
isize = edict({'h': H, 'w': W})
ref_t = nframes // 2
npix = H * W
# -- groundtruth flow --
flow_gt = repeat(flow_gt, 'i tm1 two -> i p tm1 two', p=npix)
print("sample['flow']: ", flow_gt.shape)
aligned_of = align_from_flow(dyn_clean,
flow_gt,
patchsize,
isize=isize)
# -- compute nearest neighbor fields --
shape_str = 't b h w two -> b (h w) t two'
nnf_vals, nnf_pix = nnf.compute_burst_nnf(dyn_clean, ref_t, patchsize)
nnf_pix_best = torch.LongTensor(
rearrange(nnf_pix[..., 0, :], shape_str))
nnf_pix_best = torch.LongTensor(nnf_pix_best)
flow_nnf = pix_to_flow(nnf_pix_best)
aligned_nnf = align_from_pix(dyn_clean, nnf_pix_best, patchsize)
# -- compute proposed search of nnf --
flow_split = optim.v1.run_image_burst(dyn_clean, patchsize, evaluator,
nblocks, iterations, subsizes, K)
isize = edict({'h': H, 'w': W})
aligned_split = align_from_flow(dyn_clean,
flow_split,
patchsize,
isize=isize)
# -- compute proposed search of nnf --
flow_est = optim.v3.run_image_burst(dyn_clean, patchsize, evaluator,
nblocks, iterations, subsizes, K)
aligned_est = align_from_flow(dyn_clean,
flow_est,
patchsize,
isize=isize)
# -- banner --
print("-" * 25 + " Results " + "-" * 25)
# -- compare gt v.s. nnf computations --
nnf_of = compute_epe(flow_nnf, flow_gt)
split_of = compute_epe(flow_split, flow_gt)
est_of = compute_epe(flow_est, flow_gt)
split_nnf = compute_epe(flow_split, flow_nnf)
est_nnf = compute_epe(flow_est, flow_nnf)
print("-" * 50)
print("EPE Errors")
print("-" * 50)
print("NNF v.s. Optical Flow.")
print(nnf_of)
print("Split v.s. Optical Flow.")
print(split_of)
print("Proposed v.s. Optical Flow.")
print(est_of)
print("Split v.s. NNF")
print(split_nnf)
print("Proposed v.s. NNF")
print(est_nnf)
# -- psnr eval --
pad = 2 * patchsize
isize = edict({'h': H - pad, 'w': W - pad})
psnr_of = compute_aligned_psnr(aligned_of, static_clean, isize)
psnr_nnf = compute_aligned_psnr(aligned_nnf, static_clean, isize)
psnr_split = compute_aligned_psnr(aligned_split, static_clean, isize)
psnr_est = compute_aligned_psnr(aligned_est, static_clean, isize)
print("-" * 50)
print("PSNR Values")
print("-" * 50)
print("Optical Flow [groundtruth v1]")
print(psnr_of)
print("NNF [groundtruth v2]")
print(psnr_nnf)
print("Split [old method]")
print(psnr_split)
print("Proposed [new method]")
print(psnr_est)
def execute_experiment(cfg):
# -- init exp! --
print("RUNNING EXP.")
print(cfg)
# -- create results record to save --
dims = {
'batch_results': None,
'batch_to_record': None,
'record_results': {
'default': 0
},
'stack': {
'default': 0
},
'cat': {
'default': 0
}
}
record = cache_io.ExpRecord(dims)
# -- set random seed --
set_seed(cfg.random_seed)
# -- load dataset --
data, loaders = load_image_dataset(cfg)
image_iter = iter(loaders.tr)
# -- get score function --
score_fxn_ave = get_score_function("ave")
score_fxn_mse = get_score_function("mse")
score_fxn_bs = get_score_function("bootstrapping")
score_fxn_bs_cf = get_score_function("bootstrapping_cf")
# score_fxn_bs = get_score_function("bootstrapping_mod2")
# score_fxn_bs = get_score_function(cfg.score_fxn_name)
score_fxn_bsl = get_score_function("bootstrapping_limitB")
# -- some constants --
NUM_BATCHES = 3
nframes, nblocks = cfg.nframes, cfg.nblocks
patchsize = cfg.patchsize
ppf = cfg.dynamic_info.ppf
check_parameters(nblocks, patchsize)
# -- create evaluator for ave; simple --
iterations, K = 1, 1
subsizes = []
block_batchsize = 256
eval_ave_simp = EvalBlockScores(score_fxn_ave, "ave", patchsize,
block_batchsize, None)
# -- create evaluator for ave --
iterations, K = 1, 1
subsizes = []
eval_ave = EvalBlockScores(score_fxn_ave, "ave", patchsize,
block_batchsize, None)
eval_mse = EvalBlockScores(score_fxn_mse, "mse", patchsize,
block_batchsize, None)
# -- create evaluator for bootstrapping --
block_batchsize = 81
eval_plimb = EvalBootBlockScores(score_fxn_bsl, score_fxn_bs, "bsl",
patchsize, block_batchsize, None)
eval_prop = EvalBlockScores(score_fxn_bs, "bs", patchsize, block_batchsize,
None)
eval_prop_cf = EvalBlockScores(score_fxn_bs_cf, "bs_cf", patchsize,
block_batchsize, None)
# -- iterate over images --
for image_bindex in range(NUM_BATCHES):
print("-=" * 30 + "-")
print(f"Running image batch index: {image_bindex}")
print("-=" * 30 + "-")
torch.cuda.empty_cache()
# -- sample & unpack batch --
sample = next(image_iter)
sample_to_cuda(sample)
dyn_noisy = sample['noisy'] # dynamics and noise
dyn_clean = sample['burst'] # dynamics and no noise
static_noisy = sample['snoisy'] # no dynamics and noise
static_clean = sample['sburst'] # no dynamics and no noise
flow_gt = sample['flow']
image_index = sample['index']
tl_index = sample['tl_index']
rng_state = sample['rng_state']
if cfg.noise_params.ntype == "pn":
dyn_noisy = anscombe.forward(dyn_noisy)
# dyn_nosiy = dyn_clean
dyn_noisy = static_clean
# -- shape info --
T, B, C, H, W = dyn_noisy.shape
isize = edict({'h': H, 'w': W})
ref_t = nframes // 2
nimages, npix, nframes = B, H * W, T
# -- create results dict --
flows = edict()
aligned = edict()
runtimes = edict()
optimal_scores = edict() # score function at optimal
# -- groundtruth flow --
flow_gt_rs = rearrange(flow_gt, 'i tm1 two -> i 1 tm1 two')
blocks_gt = flow_to_blocks(flow_gt_rs, nblocks)
flows.of = repeat(flow_gt, 'i tm1 two -> i p tm1 two', p=npix)
aligned.of = align_from_flow(dyn_clean, flows.of, nblocks, isize=isize)
runtimes.of = 0. # given
optimal_scores.of = np.zeros(
(nimages, npix, nframes)) # clean target is zero
aligned.clean = static_clean
# -- compute nearest neighbor fields [global] --
start_time = time.perf_counter()
shape_str = 't b h w two -> b (h w) t two'
nnf_vals, nnf_pix = nnf.compute_burst_nnf(dyn_clean, ref_t, patchsize)
nnf_pix_best = torch.LongTensor(
rearrange(nnf_pix[..., 0, :], shape_str))
nnf_pix_best = torch.LongTensor(nnf_pix_best)
flows.nnf = pix_to_flow(nnf_pix_best)
aligned.nnf = align_from_pix(dyn_clean, nnf_pix_best, nblocks)
runtimes.nnf = time.perf_counter() - start_time
optimal_scores.nnf = np.zeros(
(nimages, npix, nframes)) # clean target is zero
# -- compute proposed search of nnf --
print("[Bootstrap] loss function")
iterations = 1
K, subsizes = get_boot_hyperparams(cfg.nframes, cfg.nblocks)
start_time = time.perf_counter()
optim = AlignOptimizer("v3")
# flows.est = optim.run(dyn_noisy,patchsize,eval_prop,
# nblocks,iterations,subsizes,K)
flows.est = flows.of.clone()
aligned.est = align_from_flow(dyn_clean,
flows.est,
patchsize,
isize=isize)
runtimes.est = time.perf_counter() - start_time
# -- load adjacent blocks --
nframes, nimages, ncolor, h, w = dyn_noisy.shape
nsegs = h * w
brange = exh_block_range(nimages, nsegs, nframes, nblocks)
ref_block = get_ref_block(nblocks)
curr_blocks = init_optim_block(nimages, nsegs, nframes, nblocks)
curr_blocks = curr_blocks[:, :, :,
None] # nimages, nsegs, nframes, naligns
frames = np.r_[np.arange(nframes // 2),
np.arange(nframes // 2 + 1, nframes)]
frames = repeat(frames, 't -> i s t', i=nimages, s=nsegs)
search_blocks = get_search_blocks(frames, brange, curr_blocks,
f'cuda:{cfg.gpuid}')
print("search_blocks.shape ", search_blocks.shape)
init_blocks = rearrange(curr_blocks,
'i s t a -> i s a t').to(dyn_noisy.device)
print("init_blocks.shape ", init_blocks.shape)
search_blocks = search_blocks[0, 0]
init_blocks_ = init_blocks[0, 0]
search_blocks = eval_plimb.filter_blocks_to_1skip_neighbors(
search_blocks, init_blocks_)
search_blocks = repeat(search_blocks,
'a t -> i s a t',
i=nimages,
s=nsegs)
print("search_blocks.shape ", search_blocks.shape)
# -- compute MSE for the batch --
est = edict()
bscf = edict()
plimb = edict()
print("curr_blocks.shape ", curr_blocks.shape)
eval_prop.score_fxn_name = ""
scores, scores_t, blocks = eval_mse.score_burst_from_blocks(
dyn_noisy, search_blocks, patchsize, nblocks)
est.scores = scores
est.scores_t = scores_t
est.blocks = blocks
print("Done with est.")
# -- compute bootrapping in closed form --
scores, scores_t, blocks = eval_prop_cf.score_burst_from_blocks(
dyn_noisy, search_blocks, patchsize, nblocks)
bscf.scores = scores
bscf.scores_t = scores_t
bscf.blocks = blocks
# -- compute bootstrapping for the batch --
print("Get init block from original bootstrap.")
print(init_blocks[0, 0])
scores, scores_t, blocks = eval_prop.score_burst_from_blocks(
dyn_noisy, init_blocks, patchsize, nblocks)
print("Starting prop.")
eval_prop.score_fxn_name = "bs"
eval_prop.score_cfg.bs_type = ""
state = edict({'scores': scores, 'blocks': blocks})
scores, scores_t, blocks = eval_plimb.score_burst_from_blocks(
dyn_noisy, search_blocks, state, patchsize, nblocks)
plimb.scores = scores
plimb.scores_t = scores_t
plimb.blocks = blocks
print(est.scores.shape)
print(bscf.scores.shape)
print(state.scores.shape)
print(plimb.scores.shape)
diff_plimb = plimb.scores[0] - est.scores[0]
perc_delta = torch.abs(diff_plimb) / est.scores[0]
diff_bscf = bscf.scores[0] - est.scores[0]
perc_delta_cf = torch.abs(diff_bscf) / est.scores[0]
pix_idx_list = [0, 20, 30] #np.arange(h*w)
for p in pix_idx_list:
print("-" * 10 + f" @ {p}")
print("est", est.scores[0, p].cpu().numpy())
print("state", state.scores[0, p].cpu().numpy())
print("bscf", bscf.scores[0, p].cpu().numpy())
print("plimb", plimb.scores[0, p].cpu().numpy())
print("plimb/est", plimb.scores[0, p] / est.scores[0, p])
print("plimb - est", plimb.scores[0, p] - est.scores[0, p])
print("plimb - bscf", plimb.scores[0, p] - bscf.scores[0, p])
print("%Delta [plimb]", perc_delta[p])
print("L2-Norm [plimb]", torch.sum(diff_plimb[p]**2))
print("Nmlz L2-Norm [plimb]", torch.mean(diff_plimb[p]**2))
print("%Delta [bscf]", perc_delta_cf[p])
print("L2-Norm [bscf]", torch.sum(diff_bscf[p]**2))
print("Nmlz L2-Norm [bscf]", torch.mean(diff_bscf[p]**2))
print("[Overall: plimb] %Delta: ", torch.mean(perc_delta).item())
print("[Overall: plimb] L2-Norm: ", torch.sum(diff_plimb**2).item())
print("[Overall: plimb] Nmlz L2-Norm: ",
torch.mean(diff_plimb**2).item())
print("[Overall: bscf] %Delta: ", torch.mean(perc_delta_cf).item())
print("[Overall: bscf] L2-Norm: ", torch.sum(diff_bscf**2).item())
print("[Overall: bscf] Nmlz L2-Norm: ",
torch.mean(diff_bscf**2).item())
# -- format results --
pad = 3 #2*(nframes-1)*ppf+4
isize = edict({'h': H - pad, 'w': W - pad})
# -- flows to numpy --
is_even = cfg.frame_size % 2 == 0
mid_pix = cfg.frame_size * cfg.frame_size // 2 + (cfg.frame_size //
2) * is_even
mid_pix = 32 * 10 + 23
flows_np = edict_torch_to_numpy(flows)
# -- End-Point-Errors --
epes_of = compute_flows_epe_wrt_ref(flows, "of")
epes_nnf = compute_flows_epe_wrt_ref(flows, "nnf")
epes_nnf_local = compute_flows_epe_wrt_ref(flows, "nnf_local")
nnf_acc = compute_acc_wrt_ref(flows, "nnf")
nnf_local_acc = compute_acc_wrt_ref(flows, "nnf_local")
# -- PSNRs --
aligned = remove_frame_centers(aligned)
psnrs = compute_frames_psnr(aligned, isize)
# -- print report ---
print("\n" * 3) # banner
print("-" * 25 + " Results " + "-" * 25)
print_dict_ndarray_0_midpix(flows_np, mid_pix)
print_runtimes(runtimes)
print_verbose_psnrs(psnrs)
print_delta_summary_psnrs(psnrs)
print_verbose_epes(epes_of, epes_nnf)
print_nnf_acc(nnf_acc)
print_nnf_local_acc(nnf_local_acc)
print_summary_epes(epes_of, epes_nnf)
print_summary_psnrs(psnrs)
# -- prepare results to be appended --
psnrs = edict_torch_to_numpy(psnrs)
epes_of = edict_torch_to_numpy(epes_of)
epes_nnf = edict_torch_to_numpy(epes_nnf)
epes_nnf_local = edict_torch_to_numpy(epes_nnf_local)
nnf_acc = edict_torch_to_numpy(nnf_acc)
nnf_local_acc = edict_torch_to_numpy(nnf_local_acc)
image_index = torch_to_numpy(image_index)
batch_results = {
'runtimes': runtimes,
'optimal_scores': optimal_scores,
'psnrs': psnrs,
'epes_of': epes_of,
'epes_nnf': epes_nnf,
'epes_nnf_local': epes_nnf_local,
'nnf_acc': nnf_acc,
'nnf_local_acc': nnf_local_acc
}
# -- format results --
batch_results = flatten_internal_dict(batch_results)
format_fields(batch_results, image_index, rng_state)
print("shape check.")
for key, value in batch_results.items():
print(key, value.shape)
record.append(batch_results)
# print("\n"*3)
# print("-"*20)
# print(record.record)
# print("-"*20)
# print("\n"*3)
# record.stack_record()
record.cat_record()
# print("\n"*3)
# print("-"*20)
# print(record.record)
# print("-"*20)
print("\n" * 3)
print("\n" * 3)
print("-" * 20)
# df = pd.DataFrame().append(record.record,ignore_index=True)
for key, val in record.record.items():
print(key, val.shape)
# print(df)
print("-" * 20)
print("\n" * 3)
return record.record
def test_nnf():
# -- get config --
cfg = config()
print("Config for Testing.")
print(cfg)
# -- set random seed --
set_seed(cfg.random_seed)
# -- load dataset --
data, loaders = load_image_dataset(cfg)
image_iter = iter(loaders.tr)
nskips = 2 + 4 + 2 + 4 + 1
for skip in range(nskips):
next(image_iter)
# -- get score function --
score_fxn_ave = get_score_function("ave")
score_fxn_bs = get_score_function(cfg.score_fxn_name)
# -- some constants --
NUM_BATCHES = 10
nframes, nblocks = cfg.nframes, cfg.nblocks
patchsize = cfg.patchsize
ppf = cfg.dynamic_info.ppf
check_parameters(nblocks, patchsize)
# -- create evaluator for ave; simple --
iterations, K = 1, 1
subsizes = []
block_batchsize = 256
eval_ave_simp = EvalBlockScores(score_fxn_ave, "ave", patchsize,
block_batchsize, None)
# -- create evaluator for ave --
iterations, K = 1, 1
subsizes = []
eval_ave = EvalBlockScores(score_fxn_ave, "ave", patchsize,
block_batchsize, None)
# -- create evaluator for bootstrapping --
block_batchsize = 64
eval_prop = EvalBlockScores(score_fxn_bs, "bs", patchsize, block_batchsize,
None)
# -- iterate over images --
for image_bindex in range(NUM_BATCHES):
print("-=" * 30 + "-")
print(f"Running image batch index: {image_bindex}")
print("-=" * 30 + "-")
# -- sample & unpack batch --
sample = next(image_iter)
sample_to_cuda(sample)
dyn_noisy = sample['noisy'] # dynamics and noise
dyn_clean = sample['burst'] # dynamics and no noise
static_noisy = sample['snoisy'] # no dynamics and noise
static_clean = sample['sburst'] # no dynamics and no noise
flow_gt = sample['ref_flow']
# flow_gt = sample['seq_flow']
if cfg.noise_params.ntype == "pn":
dyn_noisy = anscombe.forward(dyn_noisy)
# -- shape info --
T, B, C, H, W = dyn_noisy.shape
isize = edict({'h': H, 'w': W})
ref_t = nframes // 2
npix = H * W
# -- groundtruth flow --
# print("flow_gt",flow_gt)
flow_gt_rs = rearrange(flow_gt, 'i tm1 two -> i 1 tm1 two')
blocks_gt = flow_to_blocks(flow_gt_rs, nblocks)
# print("\n\n")
# print("flow_gt[0,0] ",flow_gt)
# print("blocks_gt[0,0] ",blocks_gt[0,0])
flow_gt = repeat(flow_gt, 'i tm1 two -> i p tm1 two', p=npix)
aligned_of = align_from_flow(dyn_clean, flow_gt, nblocks, isize=isize)
pix_gt = flow_to_pix(flow_gt.clone(), isize=isize)
# -- compute nearest neighbor fields --
start_time = time.perf_counter()
shape_str = 't b h w two -> b (h w) t two'
nnf_vals, nnf_pix = nnf.compute_burst_nnf(dyn_clean, ref_t, patchsize)
nnf_pix_best = torch.LongTensor(
rearrange(nnf_pix[..., 0, :], shape_str))
nnf_pix_best = torch.LongTensor(nnf_pix_best)
pix_nnf = nnf_pix_best.clone()
flow_nnf = pix_to_flow(nnf_pix_best)
aligned_nnf = align_from_pix(dyn_clean, nnf_pix_best, nblocks)
time_nnf = time.perf_counter() - start_time
# -- compute proposed search of nnf --
start_time = time.perf_counter()
print(dyn_noisy.shape)
# split_vals,split_pix = nnf.compute_burst_nnf(dyn_noisy,ref_t,patchsize)
split_pix = np.copy(nnf_pix)
split_pix_best = torch.LongTensor(
rearrange(split_pix[..., 0, :], shape_str))
split_pix_best = torch.LongTensor(split_pix_best)
pix_split = split_pix_best.clone()
flow_split = pix_to_flow(split_pix_best)
aligned_split = align_from_pix(dyn_clean, split_pix_best, nblocks)
time_split = time.perf_counter() - start_time
# -- compute simple ave --
iterations, K = 0, 1
subsizes = []
print("[simple] Ave loss function")
start_time = time.perf_counter()
optim = AlignOptimizer("v3")
# flow_ave_simp = optim.run(dyn_noisy,patchsize,eval_ave_simp,
# nblocks,iterations,subsizes,K)
flow_ave_simp = flow_gt.clone().cpu()
aligned_ave_simp = align_from_flow(dyn_clean,
flow_ave_simp,
nblocks,
isize=isize)
time_ave_simp = time.perf_counter() - start_time
print(flow_ave_simp.shape)
# -- compute complex ave --
iterations, K = 0, 1
subsizes = []
print("[complex] Ave loss function")
start_time = time.perf_counter()
optim = AlignOptimizer("v3")
flow_ave = optim.run(dyn_noisy, patchsize, eval_ave, nblocks,
iterations, subsizes, K)
# flow_ave = flow_gt.clone()
pix_ave = flow_to_pix(flow_ave.clone(), isize=isize)
aligned_ave = align_from_flow(dyn_clean,
flow_ave,
nblocks,
isize=isize)
time_ave = time.perf_counter() - start_time
# -- compute proposed search of nnf --
# iterations,K = 50,3
# subsizes = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2]
#iterations,K = 1,nblocks**2
# K is a function of noise level.
# iterations,K = 1,nblocks**2
iterations, K = 1, 2 * nblocks #**2
# subsizes = [3]#,3,3,3,3,3,3,3,3,3]
# subsizes = [3,3,3,3,3,3,3,]
subsizes = [3, 3, 3, 3, 3, 3, 3, 3]
# subsizes = [nframes]
# subsizes = [nframes]
print("[Bootstrap] loss function")
start_time = time.perf_counter()
optim = AlignOptimizer("v3")
flow_est = optim.run(dyn_noisy, patchsize, eval_prop, nblocks,
iterations, subsizes, K)
pix_est = flow_to_pix(flow_est.clone(), isize=isize)
aligned_est = align_from_flow(dyn_clean,
flow_est,
patchsize,
isize=isize)
time_est = time.perf_counter() - start_time
# flow_est = flow_gt.clone()
# aligned_est = aligned_of.clone()
# time_est = 0.
# -- banner --
print("\n" * 3)
print("-" * 25 + " Results " + "-" * 25)
# -- examples of flow --
print("-" * 50)
is_even = cfg.frame_size % 2 == 0
mid_pix = cfg.frame_size * cfg.frame_size // 2 + (cfg.frame_size //
2) * is_even
mid_pix = 32 * 10 + 23
# mid_pix = 32*23+10
flow_gt_np = torch_to_numpy(flow_gt)
flow_nnf_np = torch_to_numpy(flow_nnf)
flow_split_np = torch_to_numpy(flow_split)
flow_ave_simp_np = torch_to_numpy(flow_ave_simp)
flow_ave_np = torch_to_numpy(flow_ave)
flow_est_np = torch_to_numpy(flow_est)
print(flow_gt_np[0, mid_pix])
print(flow_nnf_np[0, mid_pix])
print(flow_split_np[0, mid_pix])
print(flow_ave_simp_np[0, mid_pix])
print(flow_ave_np[0, mid_pix])
print(flow_est_np[0, mid_pix])
print("-" * 50)
pix_gt_np = torch_to_numpy(pix_gt)
pix_nnf_np = torch_to_numpy(pix_nnf)
pix_ave_np = torch_to_numpy(pix_ave)
pix_est_np = torch_to_numpy(pix_est)
print(pix_gt_np[0, mid_pix])
print(pix_nnf_np[0, mid_pix])
print(pix_ave_np[0, mid_pix])
print(pix_est_np[0, mid_pix])
# print(aligned_of[0,0,:,10,23].cpu() - static_clean[0,0,:,10,23].cpu())
# print(aligned_ave[0,0,:,10,23].cpu() - static_clean[0,0,:,10,23].cpu())
# print(aligned_of[0,0,:,23,10].cpu() - static_clean[0,0,:,23,10].cpu())
# print(aligned_ave[0,0,:,23,10].cpu() - static_clean[0,0,:,23,10].cpu())
print("-" * 50)
# -- compare compute time --
print("-" * 50)
print("Compute Time [smaller is better]")
print("-" * 50)
print("[NNF]: %2.3e" % time_nnf)
print("[Split]: %2.3e" % time_split)
print("[Ave [Simple]]: %2.3e" % time_ave_simp)
print("[Ave]: %2.3e" % time_ave)
print("[Proposed]: %2.3e" % time_est)
# -- compare gt v.s. nnf computations --
nnf_of = compute_epe(flow_nnf, flow_gt)
split_of = compute_epe(flow_split, flow_gt)
ave_simp_of = compute_epe(flow_ave_simp, flow_gt)
ave_of = compute_epe(flow_ave, flow_gt)
est_of = compute_epe(flow_est, flow_gt)
split_nnf = compute_epe(flow_split, flow_nnf)
ave_simp_nnf = compute_epe(flow_ave_simp, flow_nnf)
ave_nnf = compute_epe(flow_ave, flow_nnf)
est_nnf = compute_epe(flow_est, flow_nnf)
# -- End-Point-Errors --
print("-" * 50)
print("EPE Errors [smaller is better]")
print("-" * 50)
print("NNF v.s. Optical Flow.")
print(nnf_of)
print("Split v.s. Optical Flow.")
print(split_of)
print("Ave [Simple] v.s. Optical Flow.")
print(ave_simp_of)
print("Ave v.s. Optical Flow.")
print(ave_of)
print("Proposed v.s. Optical Flow.")
print(est_of)
print("Split v.s. NNF")
print(split_nnf)
print("Ave [Simple] v.s. NNF")
print(ave_simp_nnf)
print("Ave v.s. NNF")
print(ave_nnf)
print("Proposed v.s. NNF")
print(est_nnf)
# -- compare accuracy of method nnf v.s. actual nnf --
def compute_flow_acc(guess, gt):
both = torch.all(guess.type(torch.long) == gt.type(torch.long),
dim=-1)
ncorrect = torch.sum(both)
acc = 100 * float(ncorrect) / both.numel()
return acc
split_nnf_acc = compute_flow_acc(flow_split, flow_nnf)
ave_simp_nnf_acc = compute_flow_acc(flow_ave_simp, flow_nnf)
ave_nnf_acc = compute_flow_acc(flow_ave, flow_nnf)
est_nnf_acc = compute_flow_acc(flow_est, flow_nnf)
# -- PSNR to Reference Image --
pad = 2 * (nframes - 1) * ppf + 4
isize = edict({'h': H - pad, 'w': W - pad})
# print("isize: ",isize)
aligned_of = remove_center_frame(aligned_of)
aligned_nnf = remove_center_frame(aligned_nnf)
aligned_split = remove_center_frame(aligned_split)
aligned_ave_simp = remove_center_frame(aligned_ave_simp)
aligned_ave = remove_center_frame(aligned_ave)
aligned_est = remove_center_frame(aligned_est)
static_clean = remove_center_frame(static_clean)
psnr_of = compute_aligned_psnr(aligned_of, static_clean, isize)
psnr_nnf = compute_aligned_psnr(aligned_nnf, static_clean, isize)
psnr_split = compute_aligned_psnr(aligned_split, static_clean, isize)
psnr_ave_simp = compute_aligned_psnr(aligned_ave_simp, static_clean,
isize)
psnr_ave = compute_aligned_psnr(aligned_ave, static_clean, isize)
psnr_est = compute_aligned_psnr(aligned_est, static_clean, isize)
print("-" * 50)
print("PSNR Values [bigger is better]")
print("-" * 50)
print("Optical Flow [groundtruth v1]")
print(psnr_of)
print("NNF [groundtruth v2]")
print(psnr_nnf)
print("Split [old method]")
print(psnr_split)
print("Ave [simple; old method]")
print(psnr_ave_simp)
print("Ave [old method]")
print(psnr_ave)
print("Proposed [new method]")
print(psnr_est)
# -- print nnf accuracy here --
print("-" * 50)
print("NNF Accuracy [bigger is better]")
print("-" * 50)
print("Split v.s. NNF")
print(split_nnf_acc)
print("Ave [Simple] v.s. NNF")
print(ave_simp_nnf_acc)
print("Ave v.s. NNF")
print(ave_nnf_acc)
print("Proposed v.s. NNF")
print(est_nnf_acc)
# -- location of PSNR errors --
csize = 30
# aligned_of = torch_to_numpy(tvF.center_crop(aligned_of,(csize,csize)))
# aligned_ave = torch_to_numpy(tvF.center_crop(aligned_ave,(csize,csize)))
# static_clean = torch_to_numpy(tvF.center_crop(static_clean,(csize,csize)))
flow_gt = torch_to_numpy(flow_gt)
flow_ave = torch_to_numpy(flow_ave)
aligned_of = torch_to_numpy(aligned_of)
aligned_ave = torch_to_numpy(aligned_ave)
static_clean = torch_to_numpy(static_clean)
# print("WHERE?")
# print("OF")
# print(aligned_of.shape)
# for row in range(30):
# print(np.abs(aligned_of[0,0,0,row]- static_clean[0,0,0,row]))
# print(np.where(~np.isclose(aligned_of,aligned_of)))
# print(np.where(~np.isclose(flow_gt,flow_ave)))
# print(np.where(~np.isclose(aligned_of,aligned_of)))
# print(np.where(~np.isclose(aligned_of,static_clean)))
# print("Ave")
# indices = np.where(~np.isclose(aligned_ave,static_clean))
# row,col = indices[-2:]
# for elem in range(len(row)):
# print(np.c_[row,col][elem])
# print(np.where(~np.isclose(aligned_ave,static_clean)))
# -- Summary of End-Point-Errors --
print("-" * 50)
print("Summary of EPE Errors [smaller is better]")
print("-" * 50)
print("[NNF v.s. Optical Flow]: %2.3f" % nnf_of.mean().item())
print("[Split v.s. Optical Flow]: %2.3f" % split_of.mean().item())
print("[Ave [Simple] v.s. Optical Flow]: %2.3f" %
ave_simp_of.mean().item())
print("[Ave v.s. Optical Flow]: %2.3f" % ave_of.mean().item())
print("[Proposed v.s. Optical Flow]: %2.3f" % est_of.mean().item())
print("[Split v.s. NNF]: %2.3f" % split_nnf.mean().item())
print("[Ave [Simple] v.s. NNF]: %2.3f" % ave_simp_nnf.mean().item())
print("[Ave v.s. NNF]: %2.3f" % ave_nnf.mean().item())
print("[Proposed v.s. NNF]: %2.3f" % est_nnf.mean().item())
# -- Summary of PSNR to Reference Image --
print("-" * 50)
print("Summary PSNR Values [bigger is better]")
print("-" * 50)
print("[Optical Flow]: %2.3f" % psnr_of.mean().item())
print("[NNF]: %2.3f" % psnr_nnf.mean().item())
print("[Split]: %2.3f" % psnr_split.mean().item())
print("[Ave [Simple]]: %2.3f" % psnr_ave_simp.mean().item())
print("[Ave]: %2.3f" % psnr_ave.mean().item())
print("[Proposed]: %2.3f" % psnr_est.mean().item())
print("-" * 50)
print("PSNR Comparisons [smaller is better]")
print("-" * 50)
delta_split = psnr_nnf - psnr_split
delta_ave_simp = psnr_nnf - psnr_ave_simp
delta_ave = psnr_nnf - psnr_ave
delta_est = psnr_nnf - psnr_est
print("ave([NNF] - [Split]): %2.3f" % delta_split.mean().item())
print("ave([NNF] - [Ave [Simple]]): %2.3f" %
delta_ave_simp.mean().item())
print("ave([NNF] - [Ave]): %2.3f" % delta_ave.mean().item())
print("ave([NNF] - [Proposed]): %2.3f" % delta_est.mean().item())
def run_with_seed(seed):
# -- settings --
cfg = get_cfg_defaults()
cfg.use_anscombe = False
cfg.noise_params.ntype = 'g'
cfg.noise_params.g.std = 10.
cfg.nframes = 5
cfg.patchsize = 11
# -- seeds --
cfg.seed = seed
# cfg.seed = 123 # sky of a forest
# cfg.seed = 345 # handrail and stairs
# cfg.seed = 567 # cloudy blue sky
# cfg.seed = 567 # cloudy blue sky
# -- set seed --
set_seed(cfg.seed)
# -- load dataset --
data, loaders = load_image_dataset(cfg)
train_iter = iter(loaders.tr)
# -- fetch sample --
sample = next(train_iter)
sample_to_cuda(sample)
# -- unpack data --
noisy, clean = sample['noisy'], sample['burst']
nframes, nimages, ncolors, H, W = noisy.shape
isize = edict({'h': H, 'w': W})
# -- boxes for plotting --
boxes = edict()
aligned = edict()
# -- compute clean nnf --
vprint("[start] clean nnf.")
align_fxn = get_align_method(cfg, "l2_global")
aligned.gt, flow = align_fxn(clean, None, None)
boxes.gt = boxes_from_flow(flow, H, W)
vprint("[done] clean nnf.")
# -- compute nnf --
vprint("[start] global nnf.")
align_fxn = get_align_method(cfg, "l2_global")
_, flow = align_fxn(noisy, None, None)
aligned.global_l2 = align_from_flow(clean, flow, cfg.nblocks, isize=isize)
boxes.global_l2 = boxes_from_flow(flow, H, W)
vprint("[done] global nnf.")
# -- compute local nnf --
vprint("[start] local nnf.")
align_fxn = get_align_method(cfg, "l2_local")
_, flow = align_fxn(noisy, None, None)
aligned.local_l2 = align_from_flow(clean, flow, cfg.nblocks, isize=isize)
boxes.local_l2 = boxes_from_flow(flow, H, W)
vprint("[done] local nnf.")
# -- compute proposed score --
vprint("[start] bootstrapping.")
align_fxn = get_align_method(cfg, "bs_local_v2")
_, flow = align_fxn(noisy, None, None)
aligned.local_bs = align_from_flow(clean, flow, cfg.nblocks, isize=isize)
boxes.local_bs = boxes_from_flow(flow, H, W)
vprint("[done] bootstrapping.")
# -- reshape to image --
noisy = rearrange(noisy, 't b c h w -> t b h w c')
clean = rearrange(clean, 't b c h w -> t b h w c')
# -- normalize to [0,1] --
noisy -= noisy.min()
clean -= clean.min()
noisy /= noisy.max()
clean /= clean.max()
# -- clamp to [0,1] --
# noisy = noisy.clamp(0,1)
# clean = clean.clamp(0,1)
# print_tensor_stats("noisy",noisy)
# print_tensor_stats("clean",clean)
# -- cuda to cpu --
noisy = noisy.cpu()
clean = clean.cpu()
for field in boxes.keys():
boxes[field] = boxes[field].cpu().numpy()
# -- plot boxes for middle pix --
ref_pix = edict({'x': H // 2, 'y': W // 2})
field = 'global_l2'
plot_boxes(noisy, clean, aligned, field, boxes, ref_pix, cfg.patchsize,
seed)
def run_with_seed(seed):
# -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
#
# Settings
#
# -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
# -- settings --
cfg = get_cfg_defaults()
cfg.use_anscombe = False
cfg.noise_params.ntype = 'g'
cfg.noise_params.g.std = 25.
cfg.nframes = 3
cfg.dynamic_info.nframes = cfg.nframes
cfg.nblocks = 3
cfg.patchsize = 11
cfg.gpuid = 1
cfg.device = f"cuda:{cfg.gpuid}"
# -- seeds --
cfg.seed = seed
# cfg.seed = 123 # sky of a forest
# cfg.seed = 345 # handrail and stairs
# cfg.seed = 567 # cloudy blue sky
# cfg.seed = 567 # cloudy blue sky
# -- set seed --
set_seed(cfg.seed)
# -- load dataset --
data, loaders = load_image_dataset(cfg)
train_iter = iter(loaders.tr)
# -- fetch sample --
sample = next(train_iter)
sample_to_cuda(sample)
# -- unpack data --
noisy, clean = sample['noisy'], sample['burst']
nframes, nimages, ncolors, H, W = noisy.shape
isize = edict({'h': H, 'w': W})
# -- setup results --
scores = edict()
scores.noisy = edict()
scores.clean = edict()
blocks = edict()
blocks.noisy = edict()
blocks.clean = edict()
# -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
#
# Setup For Searches
#
# -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
# -- tile image to patches --
pad = 2 * (cfg.nblocks // 2)
h, w = cfg.patchsize + pad, cfg.patchsize + pad
noisy_patches = tile_patches(noisy, cfg.patchsize + pad).pix
noisy_patches = rearrange(noisy_patches,
'b t s (h w c) -> b s t c h w',
h=h,
w=w)
nimages, npix, nframes, c, psH, psW = noisy_patches.shape
clean_patches = tile_patches(clean, cfg.patchsize + pad).pix
clean_patches = rearrange(clean_patches,
'b t s (h w c) -> b s t c h w',
h=h,
w=w)
nimages, npix, nframes, c, psH, psW = clean_patches.shape
masks = torch.ones(nimages, npix, nframes, c, psH, psW).to(cfg.device)
# -- create constants --
frames = np.r_[np.arange(cfg.nframes // 2),
np.arange(cfg.nframes // 2 + 1, cfg.nframes)]
frames = repeat(frames, 'z -> i s z', i=nimages, s=npix)
brange = exh_block_range(nimages, npix, cfg.nframes, cfg.nblocks)
curr_blocks = init_optim_block(nimages, npix, cfg.nframes, cfg.nblocks)
srch_blocks = get_search_blocks(frames, brange, curr_blocks, cfg.device)
np_srch_blocks = torch_to_numpy(srch_blocks[0])
S = len(srch_blocks[0, 0])
# -- create constants --
frames_pair = np.array([0])
frames = repeat(frames_pair, 'z -> i s z', i=nimages, s=npix)
brange = exh_block_range(nimages, npix, cfg.nframes, cfg.nblocks)
curr_blocks_pair = init_optim_block(nimages, npix, cfg.nframes,
cfg.nblocks)
srch_blocks_pair = get_search_blocks(frames, brange, curr_blocks_pair,
cfg.device)
S_pair = len(srch_blocks[0, 0])
# -- encode blocks --
single_search_block = srch_blocks[0, 0].cpu().numpy()
block_strings = search_blocks_to_str(single_search_block)
labels = search_blocks_to_labels(single_search_block, block_strings)
# -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
#
# Execute Searches
#
# -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
#
# --- run PAIRED split search ---
#
ave_fxn = get_score_function("ave")
block_batchsize = 128
evaluator = EvalBlockScores(ave_fxn, "ave", cfg.patchsize, block_batchsize,
None)
get_topK = evaluator.compute_topK_scores
# -- a) run clean --
clean_scores, clean_blocks = get_topK(clean_patches, masks,
srch_blocks_pair, cfg.nblocks,
S_pair)
scores_full = torch_to_numpy(clean_scores[0])
blocks_full = torch_to_numpy(clean_blocks[0])
# -- b) tile results to full blocks --
scores_full, blocks_full = tile_pair_to_full(scores_full, blocks_full,
np_srch_blocks, frames_pair,
cfg.nframes, cfg.nblocks)
scores.clean.ave = scores_full
blocks.clean.ave = batch_search_blocks_to_labels(blocks_full,
block_strings)
# -- a) run noisy --
noisy_scores, noisy_blocks = get_topK(noisy_patches, masks,
srch_blocks_pair, cfg.nblocks,
S_pair)
scores_full = torch_to_numpy(noisy_scores[0])
blocks_full = torch_to_numpy(noisy_blocks[0])
# -- b) tile results to full blocks --
scores_full, blocks_full = tile_pair_to_full(scores_full, blocks_full,
np_srch_blocks, frames_pair,
cfg.nframes, cfg.nblocks)
scores.noisy.ave = scores_full
blocks.noisy.ave = batch_search_blocks_to_labels(blocks_full,
block_strings)
#
# --- run FULL split search ---
#
ave_fxn = get_score_function("ave")
block_batchsize = 128
evaluator = EvalBlockScores(ave_fxn, "ave", cfg.patchsize, block_batchsize,
None)
get_topK = evaluator.compute_topK_scores
# -- run clean --
clean_scores, clean_blocks = get_topK(clean_patches, masks, srch_blocks,
cfg.nblocks, S)
clean_scores = torch_to_numpy(clean_scores)
scores.clean.full_ave = clean_scores[0]
clean_blocks = torch_to_numpy(clean_blocks)
batch_blocks = clean_blocks[0, :, :, :]
blocks.clean.full_ave = batch_search_blocks_to_labels(
batch_blocks, block_strings)
# -- run noisy --
noisy_scores, noisy_blocks = get_topK(noisy_patches, masks, srch_blocks,
cfg.nblocks, S)
noisy_scores = torch_to_numpy(noisy_scores)
scores.noisy.full_ave = noisy_scores[0]
noisy_blocks = torch_to_numpy(noisy_blocks)
batch_blocks = noisy_blocks[0, :, :, :]
blocks.noisy.full_ave = batch_search_blocks_to_labels(
batch_blocks, block_strings)
#
# --- run bootstrapping ---
#
bs_fxn = get_score_function("bootstrapping_mod2")
block_batchsize = 32
evaluator = EvalBlockScores(bs_fxn, "bs_mod2", cfg.patchsize,
block_batchsize, None)
get_topK = evaluator.compute_topK_scores
# -- run noisy --
noisy_scores, noisy_blocks = get_topK(noisy_patches, masks, srch_blocks,
cfg.nblocks, S)
noisy_scores = torch_to_numpy(noisy_scores)
scores.noisy.bs = noisy_scores[0]
noisy_blocks = torch_to_numpy(noisy_blocks)
batch_blocks = noisy_blocks[0, :, :, :]
blocks.noisy.bs = batch_search_blocks_to_labels(batch_blocks,
block_strings)
# -- run clean --
clean_scores, clean_blocks = get_topK(clean_patches, masks, srch_blocks,
cfg.nblocks, S)
clean_scores = torch_to_numpy(clean_scores)
scores.clean.bs = clean_scores[0]
clean_blocks = torch_to_numpy(clean_blocks)
batch_blocks = noisy_blocks[0, :, :, :]
blocks.clean.bs = batch_search_blocks_to_labels(batch_blocks,
block_strings)
# -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
#
# Plot Results
#
# -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-
print("Plotting Results.")
plot_landscape(scores, blocks, seed)
