def align_from_flow_notimpl(burst,flow,nblocks): r""" Align a burst of frames from a pixel tensor representing each pixel's adjustment """ # -- shapes -- nframes,nimages,c,h,w = burst.shape nimages,npix,nframes_m1,two = flow.shape nframes = nframes_m1 + 1 # -- add paddings -- burst = rearrange(burst,'t i c h w -> (t i) c h w') pad_burst = F.pad(burst,[nblocks//2,]*4,mode='reflect') pad_burst = rearrange(pad_burst,'(t i) c h w -> t i c h w',i=nimages) # -- ensure ndarray -- pad_burst = torch_to_numpy(pad_burst) flow = torch_to_numpy(flow) # -- create blocks -- aligned = np.zeros((nframes,nimages,c,h,w)).astype(np.float64) pad_burst = pad_burst.astype(np.float64) align_from_flow_numba(aligned,pad_burst,flow,nblocks) # -- back to torch -- aligned = torch.FloatTensor(aligned) aligned[nframes//2] = burst[nframes//2].clone() return aligned
def split_frame_search_serial(patches,masks,evaluator,curr_blocks,brange,nblocks,K): r""" brange: a list of search ranges for each frame """ # -- shapes and init -- nimages,nsegs,nframes = patches.shape[:3] ones = np.ones((nimages,nsegs,1)) topK_blocks = init_topK_split_search(nimages,nsegs,nframes,K) assert nimages == 1,"Only batchsize 1 right now." ref_block = get_ref_block(nblocks) device = patches.device brange = torch_to_numpy(brange) curr_blocks = torch_to_numpy(curr_blocks) for t in range(nframes): if t == nframes//2: topK_blocks[:,:,t,:] = ref_block continue blocks_t = ones * t bsize = evaluator.block_batchsize srch_blocks = get_search_blocks(blocks_t,brange,curr_blocks,device,False) # srch_blocks = mesh_block_ranges_gen(blocks_t,brange,curr_blocks,bsize,device) # srch_blocks = mesh_block_ranges(blocks_t,brange,curr_blocks,device) scores,scores_t,blocks = evaluator.compute_topK(patches,masks, srch_blocks,nblocks,K) # blocks.shape = nimages, nsegs, K, nframes topK_blocks_t = blocks[:,:,:,t] topK_blocks[:,:,t,:] = topK_blocks_t torch.cuda.empty_cache() return topK_blocks
def rand_subset_search(patches, masks, evaluator, curr_blocks, brange, nblocks, subsizes): r""" brange: a list of search ranges for each frame """ # -- skip of subset sizes provided is empty -- if len(subsizes) == 0: return torch_to_numpy(curr_blocks) # -- this would be a good place for particle filtering -- # -- 1.) for each particle we would search along random subsets -- # -- 2.) merge results periodically (e.g. take best one so far) -- # -- 3.) alternate between (1) and (2) device = patches.device nimages, nsegs, nframes = patches.shape[:3] brange = torch_to_numpy(brange) curr_blocks = torch_to_numpy(curr_blocks) for size in subsizes: ps = evaluator.patchsize # frames = propose_frames(patches,curr_blocks,nframes,ps,size,nblocks) rands = npr.choice(nframes, size=size, replace=False) frames = repeat(rands, 'z -> i s z', i=nimages, s=nsegs) srch_blocks = get_search_blocks(frames, brange, curr_blocks, device) scores, scores_t, blocks = evaluator.compute_topK( patches, masks, srch_blocks, nblocks, 1) blocks = torch_to_numpy(blocks) curr_blocks = blocks[:, :, 0, :] torch.cuda.empty_cache() return curr_blocks
def compute_bootstrap(samples, scores_t, counts_t, ave, subsets, nbatches, batchsize): samples = torch_to_numpy(samples) scores_t = torch_to_numpy(scores_t) counts_t = torch_to_numpy(counts_t) ave = torch_to_numpy(ave) subsets = torch_to_numpy(subsets) bootstrap_numba(samples, scores_t, counts_t, ave, subsets, nbatches, batchsize)
def split_frame_search_parallel(patches,masks,evaluator,curr_blocks,brange,nblocks,K): # -- to numpy for meshgrid -- device = patches.device patches = torch_to_numpy(patches) brange = torch_to_numpy(brange) curr_blocks = torch_to_numpy(curr_blocks) # -- shapes and init -- nimages,nsegs,nframes = patches.shape[:3] ones = np.ones((nimages,nsegs,1)) topK_blocks = init_topK_split_search(nimages,nsegs,nframes,K) assert nimages == 1,"Only batchsize 1 right now." # -- run split frame in parallel -- args = [device,patches,masks,evaluator,ones,brange,curr_blocks,nblocks,K,topK_blocks] pParallel = ProgressParallel(use_tqdm=False,total=nframes,n_jobs=4) delayed_fxn = delayed(split_frame_search_single) pParallel(delayed_fxn(t,*args) for t in range(nframes)) # blocks.shape = nimages, nsegs, nframes, K return topK_blocks
def align_from_pix(burst,pix,pad): r""" Align a burst of frames from a pixel tensor representing each pixel's adjustment In this code, we conflate the search radius with the padding of an image. The search radius was originally a local jitter, so padding the image border removed out-of-bounds access. However, for a general search pattern this padding does not remove out-of-bounds accesses. So the "nblocks" (now renamed "pad") is actually just padding. """ # -- shapes -- nframes,nimages,c,h,w = burst.shape nimages,npix,nframes,two = pix.shape # -- add paddings -- burst_pad = rearrange(burst,'t i c h w -> (t i) c h w') pad_burst = F.pad(burst_pad,[pad//2,]*4,mode='reflect') pad_burst = rearrange(pad_burst,'(t i) c h w -> t i c h w',i=nimages) # -- ensure ndarray -- pad_burst = torch_to_numpy(pad_burst).astype(np.float64) pix = torch_to_numpy(pix) # -- create blocks -- aligned = np.zeros((nframes,nimages,c,h,w)).astype(np.float64) pad_burst = pad_burst.astype(np.float64) align_from_pix_numba(aligned,pad_burst,pix,pad) # -- back to torch -- aligned = torch.FloatTensor(aligned) return aligned
def blocks_to_flow(blocks, nblocks, ftype='ref'): # -- required dims -- nimages, npix, nframes = blocks.shape # -- compute conversion -- blocks = torch_to_numpy(blocks) blocks = rearrange(blocks, 'i p t -> (i p) t') if ftype == 'ref': flow = blocks_to_ref_flow_numba(blocks, nblocks) elif ftype == 'seq': flow = blocks_to_seq_flow_numba(blocks, nblocks) else: raise ValueError(f"Uknown input {ftype}") flow = rearrange(flow, '(i p) tprime two -> i p tprime two', i=nimages) # -- back to torch -- flow = torch.LongTensor(flow) return flow
def flow_to_blocks(flow, nblocks, ftype="ref"): # -- check shapes -- nimages, npix, nframes_prime, two = flow.shape # -- ensure int64 ndarray -- flow = torch_to_numpy(flow) flow = flow.astype(np.int64) # -- create blocks -- flow = rearrange(flow, 'i p tm1 two -> (i p) tm1 two') if ftype == "ref": blocks = ref_flow_to_blocks_numba(flow, nblocks) elif ftype == "seq": blocks = seq_flow_to_blocks_numba(flow, nblocks) else: raise ValueError(f"Uknown flow type [{ftype}]") blocks = rearrange(blocks, '(i p) t -> i p t', i=nimages) # -- to tensor -- blocks = torch.LongTensor(blocks) return blocks
def execute_experiment(cfg): # -- init exp! -- print("RUNNING EXP.") print(cfg) # -- create results record to save -- dims = { 'batch_results': None, 'batch_to_record': None, 'record_results': { 'default': 0 }, 'stack': { 'default': 0 }, 'cat': { 'default': 0 } } record = cache_io.ExpRecord(dims) # -- set random seed -- set_seed(cfg.random_seed) # -- load dataset -- data, loaders = load_image_dataset(cfg) image_iter = iter(loaders.tr) # -- get score function -- score_fxn_ave = get_score_function("ave") score_fxn_mse = get_score_function("mse") score_fxn_bs = get_score_function("bootstrapping") score_fxn_bs_cf = get_score_function("bootstrapping_cf") # score_fxn_bs = get_score_function("bootstrapping_mod2") # score_fxn_bs = get_score_function(cfg.score_fxn_name) score_fxn_bsl = get_score_function("bootstrapping_limitB") # -- some constants -- NUM_BATCHES = 3 nframes, nblocks = cfg.nframes, cfg.nblocks patchsize = cfg.patchsize ppf = cfg.dynamic_info.ppf check_parameters(nblocks, patchsize) # -- create evaluator for ave; simple -- iterations, K = 1, 1 subsizes = [] block_batchsize = 256 eval_ave_simp = EvalBlockScores(score_fxn_ave, "ave", patchsize, block_batchsize, None) # -- create evaluator for ave -- iterations, K = 1, 1 subsizes = [] eval_ave = EvalBlockScores(score_fxn_ave, "ave", patchsize, block_batchsize, None) eval_mse = EvalBlockScores(score_fxn_mse, "mse", patchsize, block_batchsize, None) # -- create evaluator for bootstrapping -- block_batchsize = 81 eval_plimb = EvalBootBlockScores(score_fxn_bsl, score_fxn_bs, "bsl", patchsize, block_batchsize, None) eval_prop = EvalBlockScores(score_fxn_bs, "bs", patchsize, block_batchsize, None) eval_prop_cf = EvalBlockScores(score_fxn_bs_cf, "bs_cf", patchsize, block_batchsize, None) # -- iterate over images -- for image_bindex in range(NUM_BATCHES): print("-=" * 30 + "-") print(f"Running image batch index: {image_bindex}") print("-=" * 30 + "-") torch.cuda.empty_cache() # -- sample & unpack batch -- sample = next(image_iter) sample_to_cuda(sample) dyn_noisy = sample['noisy'] # dynamics and noise dyn_clean = sample['burst'] # dynamics and no noise static_noisy = sample['snoisy'] # no dynamics and noise static_clean = sample['sburst'] # no dynamics and no noise flow_gt = sample['flow'] image_index = sample['index'] tl_index = sample['tl_index'] rng_state = sample['rng_state'] if cfg.noise_params.ntype == "pn": dyn_noisy = anscombe.forward(dyn_noisy) # dyn_nosiy = dyn_clean dyn_noisy = static_clean # -- shape info -- T, B, C, H, W = dyn_noisy.shape isize = edict({'h': H, 'w': W}) ref_t = nframes // 2 nimages, npix, nframes = B, H * W, T # -- create results dict -- flows = edict() aligned = edict() runtimes = edict() optimal_scores = edict() # score function at optimal # -- groundtruth flow -- flow_gt_rs = rearrange(flow_gt, 'i tm1 two -> i 1 tm1 two') blocks_gt = flow_to_blocks(flow_gt_rs, nblocks) flows.of = repeat(flow_gt, 'i tm1 two -> i p tm1 two', p=npix) aligned.of = align_from_flow(dyn_clean, flows.of, nblocks, isize=isize) runtimes.of = 0. # given optimal_scores.of = np.zeros( (nimages, npix, nframes)) # clean target is zero aligned.clean = static_clean # -- compute nearest neighbor fields [global] -- start_time = time.perf_counter() shape_str = 't b h w two -> b (h w) t two' nnf_vals, nnf_pix = nnf.compute_burst_nnf(dyn_clean, ref_t, patchsize) nnf_pix_best = torch.LongTensor( rearrange(nnf_pix[..., 0, :], shape_str)) nnf_pix_best = torch.LongTensor(nnf_pix_best) flows.nnf = pix_to_flow(nnf_pix_best) aligned.nnf = align_from_pix(dyn_clean, nnf_pix_best, nblocks) runtimes.nnf = time.perf_counter() - start_time optimal_scores.nnf = np.zeros( (nimages, npix, nframes)) # clean target is zero # -- compute proposed search of nnf -- print("[Bootstrap] loss function") iterations = 1 K, subsizes = get_boot_hyperparams(cfg.nframes, cfg.nblocks) start_time = time.perf_counter() optim = AlignOptimizer("v3") # flows.est = optim.run(dyn_noisy,patchsize,eval_prop, # nblocks,iterations,subsizes,K) flows.est = flows.of.clone() aligned.est = align_from_flow(dyn_clean, flows.est, patchsize, isize=isize) runtimes.est = time.perf_counter() - start_time # -- load adjacent blocks -- nframes, nimages, ncolor, h, w = dyn_noisy.shape nsegs = h * w brange = exh_block_range(nimages, nsegs, nframes, nblocks) ref_block = get_ref_block(nblocks) curr_blocks = init_optim_block(nimages, nsegs, nframes, nblocks) curr_blocks = curr_blocks[:, :, :, None] # nimages, nsegs, nframes, naligns frames = np.r_[np.arange(nframes // 2), np.arange(nframes // 2 + 1, nframes)] frames = repeat(frames, 't -> i s t', i=nimages, s=nsegs) search_blocks = get_search_blocks(frames, brange, curr_blocks, f'cuda:{cfg.gpuid}') print("search_blocks.shape ", search_blocks.shape) init_blocks = rearrange(curr_blocks, 'i s t a -> i s a t').to(dyn_noisy.device) print("init_blocks.shape ", init_blocks.shape) search_blocks = search_blocks[0, 0] init_blocks_ = init_blocks[0, 0] search_blocks = eval_plimb.filter_blocks_to_1skip_neighbors( search_blocks, init_blocks_) search_blocks = repeat(search_blocks, 'a t -> i s a t', i=nimages, s=nsegs) print("search_blocks.shape ", search_blocks.shape) # -- compute MSE for the batch -- est = edict() bscf = edict() plimb = edict() print("curr_blocks.shape ", curr_blocks.shape) eval_prop.score_fxn_name = "" scores, scores_t, blocks = eval_mse.score_burst_from_blocks( dyn_noisy, search_blocks, patchsize, nblocks) est.scores = scores est.scores_t = scores_t est.blocks = blocks print("Done with est.") # -- compute bootrapping in closed form -- scores, scores_t, blocks = eval_prop_cf.score_burst_from_blocks( dyn_noisy, search_blocks, patchsize, nblocks) bscf.scores = scores bscf.scores_t = scores_t bscf.blocks = blocks # -- compute bootstrapping for the batch -- print("Get init block from original bootstrap.") print(init_blocks[0, 0]) scores, scores_t, blocks = eval_prop.score_burst_from_blocks( dyn_noisy, init_blocks, patchsize, nblocks) print("Starting prop.") eval_prop.score_fxn_name = "bs" eval_prop.score_cfg.bs_type = "" state = edict({'scores': scores, 'blocks': blocks}) scores, scores_t, blocks = eval_plimb.score_burst_from_blocks( dyn_noisy, search_blocks, state, patchsize, nblocks) plimb.scores = scores plimb.scores_t = scores_t plimb.blocks = blocks print(est.scores.shape) print(bscf.scores.shape) print(state.scores.shape) print(plimb.scores.shape) diff_plimb = plimb.scores[0] - est.scores[0] perc_delta = torch.abs(diff_plimb) / est.scores[0] diff_bscf = bscf.scores[0] - est.scores[0] perc_delta_cf = torch.abs(diff_bscf) / est.scores[0] pix_idx_list = [0, 20, 30] #np.arange(h*w) for p in pix_idx_list: print("-" * 10 + f" @ {p}") print("est", est.scores[0, p].cpu().numpy()) print("state", state.scores[0, p].cpu().numpy()) print("bscf", bscf.scores[0, p].cpu().numpy()) print("plimb", plimb.scores[0, p].cpu().numpy()) print("plimb/est", plimb.scores[0, p] / est.scores[0, p]) print("plimb - est", plimb.scores[0, p] - est.scores[0, p]) print("plimb - bscf", plimb.scores[0, p] - bscf.scores[0, p]) print("%Delta [plimb]", perc_delta[p]) print("L2-Norm [plimb]", torch.sum(diff_plimb[p]**2)) print("Nmlz L2-Norm [plimb]", torch.mean(diff_plimb[p]**2)) print("%Delta [bscf]", perc_delta_cf[p]) print("L2-Norm [bscf]", torch.sum(diff_bscf[p]**2)) print("Nmlz L2-Norm [bscf]", torch.mean(diff_bscf[p]**2)) print("[Overall: plimb] %Delta: ", torch.mean(perc_delta).item()) print("[Overall: plimb] L2-Norm: ", torch.sum(diff_plimb**2).item()) print("[Overall: plimb] Nmlz L2-Norm: ", torch.mean(diff_plimb**2).item()) print("[Overall: bscf] %Delta: ", torch.mean(perc_delta_cf).item()) print("[Overall: bscf] L2-Norm: ", torch.sum(diff_bscf**2).item()) print("[Overall: bscf] Nmlz L2-Norm: ", torch.mean(diff_bscf**2).item()) # -- format results -- pad = 3 #2*(nframes-1)*ppf+4 isize = edict({'h': H - pad, 'w': W - pad}) # -- flows to numpy -- is_even = cfg.frame_size % 2 == 0 mid_pix = cfg.frame_size * cfg.frame_size // 2 + (cfg.frame_size // 2) * is_even mid_pix = 32 * 10 + 23 flows_np = edict_torch_to_numpy(flows) # -- End-Point-Errors -- epes_of = compute_flows_epe_wrt_ref(flows, "of") epes_nnf = compute_flows_epe_wrt_ref(flows, "nnf") epes_nnf_local = compute_flows_epe_wrt_ref(flows, "nnf_local") nnf_acc = compute_acc_wrt_ref(flows, "nnf") nnf_local_acc = compute_acc_wrt_ref(flows, "nnf_local") # -- PSNRs -- aligned = remove_frame_centers(aligned) psnrs = compute_frames_psnr(aligned, isize) # -- print report --- print("\n" * 3) # banner print("-" * 25 + " Results " + "-" * 25) print_dict_ndarray_0_midpix(flows_np, mid_pix) print_runtimes(runtimes) print_verbose_psnrs(psnrs) print_delta_summary_psnrs(psnrs) print_verbose_epes(epes_of, epes_nnf) print_nnf_acc(nnf_acc) print_nnf_local_acc(nnf_local_acc) print_summary_epes(epes_of, epes_nnf) print_summary_psnrs(psnrs) # -- prepare results to be appended -- psnrs = edict_torch_to_numpy(psnrs) epes_of = edict_torch_to_numpy(epes_of) epes_nnf = edict_torch_to_numpy(epes_nnf) epes_nnf_local = edict_torch_to_numpy(epes_nnf_local) nnf_acc = edict_torch_to_numpy(nnf_acc) nnf_local_acc = edict_torch_to_numpy(nnf_local_acc) image_index = torch_to_numpy(image_index) batch_results = { 'runtimes': runtimes, 'optimal_scores': optimal_scores, 'psnrs': psnrs, 'epes_of': epes_of, 'epes_nnf': epes_nnf, 'epes_nnf_local': epes_nnf_local, 'nnf_acc': nnf_acc, 'nnf_local_acc': nnf_local_acc } # -- format results -- batch_results = flatten_internal_dict(batch_results) format_fields(batch_results, image_index, rng_state) print("shape check.") for key, value in batch_results.items(): print(key, value.shape) record.append(batch_results) # print("\n"*3) # print("-"*20) # print(record.record) # print("-"*20) # print("\n"*3) # record.stack_record() record.cat_record() # print("\n"*3) # print("-"*20) # print(record.record) # print("-"*20) print("\n" * 3) print("\n" * 3) print("-" * 20) # df = pd.DataFrame().append(record.record,ignore_index=True) for key, val in record.record.items(): print(key, val.shape) # print(df) print("-" * 20) print("\n" * 3) return record.record
def test_nnf(): # -- get config -- cfg = config() print("Config for Testing.") print(cfg) # -- set random seed -- set_seed(cfg.random_seed) # -- load dataset -- data, loaders = load_image_dataset(cfg) image_iter = iter(loaders.tr) nskips = 2 + 4 + 2 + 4 + 1 for skip in range(nskips): next(image_iter) # -- get score function -- score_fxn_ave = get_score_function("ave") score_fxn_bs = get_score_function(cfg.score_fxn_name) # -- some constants -- NUM_BATCHES = 10 nframes, nblocks = cfg.nframes, cfg.nblocks patchsize = cfg.patchsize ppf = cfg.dynamic_info.ppf check_parameters(nblocks, patchsize) # -- create evaluator for ave; simple -- iterations, K = 1, 1 subsizes = [] block_batchsize = 256 eval_ave_simp = EvalBlockScores(score_fxn_ave, "ave", patchsize, block_batchsize, None) # -- create evaluator for ave -- iterations, K = 1, 1 subsizes = [] eval_ave = EvalBlockScores(score_fxn_ave, "ave", patchsize, block_batchsize, None) # -- create evaluator for bootstrapping -- block_batchsize = 64 eval_prop = EvalBlockScores(score_fxn_bs, "bs", patchsize, block_batchsize, None) # -- iterate over images -- for image_bindex in range(NUM_BATCHES): print("-=" * 30 + "-") print(f"Running image batch index: {image_bindex}") print("-=" * 30 + "-") # -- sample & unpack batch -- sample = next(image_iter) sample_to_cuda(sample) dyn_noisy = sample['noisy'] # dynamics and noise dyn_clean = sample['burst'] # dynamics and no noise static_noisy = sample['snoisy'] # no dynamics and noise static_clean = sample['sburst'] # no dynamics and no noise flow_gt = sample['ref_flow'] # flow_gt = sample['seq_flow'] if cfg.noise_params.ntype == "pn": dyn_noisy = anscombe.forward(dyn_noisy) # -- shape info -- T, B, C, H, W = dyn_noisy.shape isize = edict({'h': H, 'w': W}) ref_t = nframes // 2 npix = H * W # -- groundtruth flow -- # print("flow_gt",flow_gt) flow_gt_rs = rearrange(flow_gt, 'i tm1 two -> i 1 tm1 two') blocks_gt = flow_to_blocks(flow_gt_rs, nblocks) # print("\n\n") # print("flow_gt[0,0] ",flow_gt) # print("blocks_gt[0,0] ",blocks_gt[0,0]) flow_gt = repeat(flow_gt, 'i tm1 two -> i p tm1 two', p=npix) aligned_of = align_from_flow(dyn_clean, flow_gt, nblocks, isize=isize) pix_gt = flow_to_pix(flow_gt.clone(), isize=isize) # -- compute nearest neighbor fields -- start_time = time.perf_counter() shape_str = 't b h w two -> b (h w) t two' nnf_vals, nnf_pix = nnf.compute_burst_nnf(dyn_clean, ref_t, patchsize) nnf_pix_best = torch.LongTensor( rearrange(nnf_pix[..., 0, :], shape_str)) nnf_pix_best = torch.LongTensor(nnf_pix_best) pix_nnf = nnf_pix_best.clone() flow_nnf = pix_to_flow(nnf_pix_best) aligned_nnf = align_from_pix(dyn_clean, nnf_pix_best, nblocks) time_nnf = time.perf_counter() - start_time # -- compute proposed search of nnf -- start_time = time.perf_counter() print(dyn_noisy.shape) # split_vals,split_pix = nnf.compute_burst_nnf(dyn_noisy,ref_t,patchsize) split_pix = np.copy(nnf_pix) split_pix_best = torch.LongTensor( rearrange(split_pix[..., 0, :], shape_str)) split_pix_best = torch.LongTensor(split_pix_best) pix_split = split_pix_best.clone() flow_split = pix_to_flow(split_pix_best) aligned_split = align_from_pix(dyn_clean, split_pix_best, nblocks) time_split = time.perf_counter() - start_time # -- compute simple ave -- iterations, K = 0, 1 subsizes = [] print("[simple] Ave loss function") start_time = time.perf_counter() optim = AlignOptimizer("v3") # flow_ave_simp = optim.run(dyn_noisy,patchsize,eval_ave_simp, # nblocks,iterations,subsizes,K) flow_ave_simp = flow_gt.clone().cpu() aligned_ave_simp = align_from_flow(dyn_clean, flow_ave_simp, nblocks, isize=isize) time_ave_simp = time.perf_counter() - start_time print(flow_ave_simp.shape) # -- compute complex ave -- iterations, K = 0, 1 subsizes = [] print("[complex] Ave loss function") start_time = time.perf_counter() optim = AlignOptimizer("v3") flow_ave = optim.run(dyn_noisy, patchsize, eval_ave, nblocks, iterations, subsizes, K) # flow_ave = flow_gt.clone() pix_ave = flow_to_pix(flow_ave.clone(), isize=isize) aligned_ave = align_from_flow(dyn_clean, flow_ave, nblocks, isize=isize) time_ave = time.perf_counter() - start_time # -- compute proposed search of nnf -- # iterations,K = 50,3 # subsizes = [2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2] #iterations,K = 1,nblocks**2 # K is a function of noise level. # iterations,K = 1,nblocks**2 iterations, K = 1, 2 * nblocks #**2 # subsizes = [3]#,3,3,3,3,3,3,3,3,3] # subsizes = [3,3,3,3,3,3,3,] subsizes = [3, 3, 3, 3, 3, 3, 3, 3] # subsizes = [nframes] # subsizes = [nframes] print("[Bootstrap] loss function") start_time = time.perf_counter() optim = AlignOptimizer("v3") flow_est = optim.run(dyn_noisy, patchsize, eval_prop, nblocks, iterations, subsizes, K) pix_est = flow_to_pix(flow_est.clone(), isize=isize) aligned_est = align_from_flow(dyn_clean, flow_est, patchsize, isize=isize) time_est = time.perf_counter() - start_time # flow_est = flow_gt.clone() # aligned_est = aligned_of.clone() # time_est = 0. # -- banner -- print("\n" * 3) print("-" * 25 + " Results " + "-" * 25) # -- examples of flow -- print("-" * 50) is_even = cfg.frame_size % 2 == 0 mid_pix = cfg.frame_size * cfg.frame_size // 2 + (cfg.frame_size // 2) * is_even mid_pix = 32 * 10 + 23 # mid_pix = 32*23+10 flow_gt_np = torch_to_numpy(flow_gt) flow_nnf_np = torch_to_numpy(flow_nnf) flow_split_np = torch_to_numpy(flow_split) flow_ave_simp_np = torch_to_numpy(flow_ave_simp) flow_ave_np = torch_to_numpy(flow_ave) flow_est_np = torch_to_numpy(flow_est) print(flow_gt_np[0, mid_pix]) print(flow_nnf_np[0, mid_pix]) print(flow_split_np[0, mid_pix]) print(flow_ave_simp_np[0, mid_pix]) print(flow_ave_np[0, mid_pix]) print(flow_est_np[0, mid_pix]) print("-" * 50) pix_gt_np = torch_to_numpy(pix_gt) pix_nnf_np = torch_to_numpy(pix_nnf) pix_ave_np = torch_to_numpy(pix_ave) pix_est_np = torch_to_numpy(pix_est) print(pix_gt_np[0, mid_pix]) print(pix_nnf_np[0, mid_pix]) print(pix_ave_np[0, mid_pix]) print(pix_est_np[0, mid_pix]) # print(aligned_of[0,0,:,10,23].cpu() - static_clean[0,0,:,10,23].cpu()) # print(aligned_ave[0,0,:,10,23].cpu() - static_clean[0,0,:,10,23].cpu()) # print(aligned_of[0,0,:,23,10].cpu() - static_clean[0,0,:,23,10].cpu()) # print(aligned_ave[0,0,:,23,10].cpu() - static_clean[0,0,:,23,10].cpu()) print("-" * 50) # -- compare compute time -- print("-" * 50) print("Compute Time [smaller is better]") print("-" * 50) print("[NNF]: %2.3e" % time_nnf) print("[Split]: %2.3e" % time_split) print("[Ave [Simple]]: %2.3e" % time_ave_simp) print("[Ave]: %2.3e" % time_ave) print("[Proposed]: %2.3e" % time_est) # -- compare gt v.s. nnf computations -- nnf_of = compute_epe(flow_nnf, flow_gt) split_of = compute_epe(flow_split, flow_gt) ave_simp_of = compute_epe(flow_ave_simp, flow_gt) ave_of = compute_epe(flow_ave, flow_gt) est_of = compute_epe(flow_est, flow_gt) split_nnf = compute_epe(flow_split, flow_nnf) ave_simp_nnf = compute_epe(flow_ave_simp, flow_nnf) ave_nnf = compute_epe(flow_ave, flow_nnf) est_nnf = compute_epe(flow_est, flow_nnf) # -- End-Point-Errors -- print("-" * 50) print("EPE Errors [smaller is better]") print("-" * 50) print("NNF v.s. Optical Flow.") print(nnf_of) print("Split v.s. Optical Flow.") print(split_of) print("Ave [Simple] v.s. Optical Flow.") print(ave_simp_of) print("Ave v.s. Optical Flow.") print(ave_of) print("Proposed v.s. Optical Flow.") print(est_of) print("Split v.s. NNF") print(split_nnf) print("Ave [Simple] v.s. NNF") print(ave_simp_nnf) print("Ave v.s. NNF") print(ave_nnf) print("Proposed v.s. NNF") print(est_nnf) # -- compare accuracy of method nnf v.s. actual nnf -- def compute_flow_acc(guess, gt): both = torch.all(guess.type(torch.long) == gt.type(torch.long), dim=-1) ncorrect = torch.sum(both) acc = 100 * float(ncorrect) / both.numel() return acc split_nnf_acc = compute_flow_acc(flow_split, flow_nnf) ave_simp_nnf_acc = compute_flow_acc(flow_ave_simp, flow_nnf) ave_nnf_acc = compute_flow_acc(flow_ave, flow_nnf) est_nnf_acc = compute_flow_acc(flow_est, flow_nnf) # -- PSNR to Reference Image -- pad = 2 * (nframes - 1) * ppf + 4 isize = edict({'h': H - pad, 'w': W - pad}) # print("isize: ",isize) aligned_of = remove_center_frame(aligned_of) aligned_nnf = remove_center_frame(aligned_nnf) aligned_split = remove_center_frame(aligned_split) aligned_ave_simp = remove_center_frame(aligned_ave_simp) aligned_ave = remove_center_frame(aligned_ave) aligned_est = remove_center_frame(aligned_est) static_clean = remove_center_frame(static_clean) psnr_of = compute_aligned_psnr(aligned_of, static_clean, isize) psnr_nnf = compute_aligned_psnr(aligned_nnf, static_clean, isize) psnr_split = compute_aligned_psnr(aligned_split, static_clean, isize) psnr_ave_simp = compute_aligned_psnr(aligned_ave_simp, static_clean, isize) psnr_ave = compute_aligned_psnr(aligned_ave, static_clean, isize) psnr_est = compute_aligned_psnr(aligned_est, static_clean, isize) print("-" * 50) print("PSNR Values [bigger is better]") print("-" * 50) print("Optical Flow [groundtruth v1]") print(psnr_of) print("NNF [groundtruth v2]") print(psnr_nnf) print("Split [old method]") print(psnr_split) print("Ave [simple; old method]") print(psnr_ave_simp) print("Ave [old method]") print(psnr_ave) print("Proposed [new method]") print(psnr_est) # -- print nnf accuracy here -- print("-" * 50) print("NNF Accuracy [bigger is better]") print("-" * 50) print("Split v.s. NNF") print(split_nnf_acc) print("Ave [Simple] v.s. NNF") print(ave_simp_nnf_acc) print("Ave v.s. NNF") print(ave_nnf_acc) print("Proposed v.s. NNF") print(est_nnf_acc) # -- location of PSNR errors -- csize = 30 # aligned_of = torch_to_numpy(tvF.center_crop(aligned_of,(csize,csize))) # aligned_ave = torch_to_numpy(tvF.center_crop(aligned_ave,(csize,csize))) # static_clean = torch_to_numpy(tvF.center_crop(static_clean,(csize,csize))) flow_gt = torch_to_numpy(flow_gt) flow_ave = torch_to_numpy(flow_ave) aligned_of = torch_to_numpy(aligned_of) aligned_ave = torch_to_numpy(aligned_ave) static_clean = torch_to_numpy(static_clean) # print("WHERE?") # print("OF") # print(aligned_of.shape) # for row in range(30): # print(np.abs(aligned_of[0,0,0,row]- static_clean[0,0,0,row])) # print(np.where(~np.isclose(aligned_of,aligned_of))) # print(np.where(~np.isclose(flow_gt,flow_ave))) # print(np.where(~np.isclose(aligned_of,aligned_of))) # print(np.where(~np.isclose(aligned_of,static_clean))) # print("Ave") # indices = np.where(~np.isclose(aligned_ave,static_clean)) # row,col = indices[-2:] # for elem in range(len(row)): # print(np.c_[row,col][elem]) # print(np.where(~np.isclose(aligned_ave,static_clean))) # -- Summary of End-Point-Errors -- print("-" * 50) print("Summary of EPE Errors [smaller is better]") print("-" * 50) print("[NNF v.s. Optical Flow]: %2.3f" % nnf_of.mean().item()) print("[Split v.s. Optical Flow]: %2.3f" % split_of.mean().item()) print("[Ave [Simple] v.s. Optical Flow]: %2.3f" % ave_simp_of.mean().item()) print("[Ave v.s. Optical Flow]: %2.3f" % ave_of.mean().item()) print("[Proposed v.s. Optical Flow]: %2.3f" % est_of.mean().item()) print("[Split v.s. NNF]: %2.3f" % split_nnf.mean().item()) print("[Ave [Simple] v.s. NNF]: %2.3f" % ave_simp_nnf.mean().item()) print("[Ave v.s. NNF]: %2.3f" % ave_nnf.mean().item()) print("[Proposed v.s. NNF]: %2.3f" % est_nnf.mean().item()) # -- Summary of PSNR to Reference Image -- print("-" * 50) print("Summary PSNR Values [bigger is better]") print("-" * 50) print("[Optical Flow]: %2.3f" % psnr_of.mean().item()) print("[NNF]: %2.3f" % psnr_nnf.mean().item()) print("[Split]: %2.3f" % psnr_split.mean().item()) print("[Ave [Simple]]: %2.3f" % psnr_ave_simp.mean().item()) print("[Ave]: %2.3f" % psnr_ave.mean().item()) print("[Proposed]: %2.3f" % psnr_est.mean().item()) print("-" * 50) print("PSNR Comparisons [smaller is better]") print("-" * 50) delta_split = psnr_nnf - psnr_split delta_ave_simp = psnr_nnf - psnr_ave_simp delta_ave = psnr_nnf - psnr_ave delta_est = psnr_nnf - psnr_est print("ave([NNF] - [Split]): %2.3f" % delta_split.mean().item()) print("ave([NNF] - [Ave [Simple]]): %2.3f" % delta_ave_simp.mean().item()) print("ave([NNF] - [Ave]): %2.3f" % delta_ave.mean().item()) print("ave([NNF] - [Proposed]): %2.3f" % delta_est.mean().item())
def execute_experiment(cfg): # -- init exp! -- print("RUNNING EXP.") print(cfg) # -- create results record to save -- dims = { 'batch_results': None, 'batch_to_record': None, 'record_results': { 'default': 0 }, 'stack': { 'default': 0 }, 'cat': { 'default': 0 } } record = cache_io.ExpRecord(dims) # -- set random seed -- set_seed(cfg.random_seed) # -- load dataset -- # data,loaders = load_image_dataset(cfg) data, loaders = load_dataset(cfg, cfg.dataset.mode) image_iter = iter(loaders.tr) # -- get score function -- score_fxn_ave = get_score_function("ave") score_fxn_bs = get_score_function(cfg.score_fxn_name) # -- some constants -- NUM_BATCHES = 10 nframes, nblocks = cfg.nframes, cfg.nblocks patchsize = cfg.patchsize ps = patchsize ppf = cfg.dynamic_info.ppf check_parameters(nblocks, patchsize) # -- theory constants -- std = cfg.noise_params.g.std / 255. p = cfg.patchsize**2 * 3 t = cfg.nframes theory = edict() theory.c2 = ((t - 1) / t)**2 * std**2 + (t - 1) / t**2 * std**2 theory.mean = theory.c2 theory.mode = (1 - 2 / p) * theory.c2 theory.var = 2 / p * theory.c2**2 theory.std = np.sqrt(theory.var) pp.pprint(theory) # npn = no patch normalization theory_npn = edict() theory_npn.c2 = ((t - 1) / t)**2 * std**2 + (t - 1) / t**2 * std**2 theory_npn.mean = theory_npn.c2 * p theory_npn.mode = (1 - 2 / p) * theory_npn.c2 * p theory_npn.var = 2 * theory_npn.c2**2 * p theory_npn.std = np.sqrt(theory_npn.var) pp.pprint(theory_npn) # oracle = clean reference frame theory_oracle = edict() theory_oracle.c2 = std**2 theory_oracle.mean = theory_oracle.c2 * p theory_oracle.mode = (1 - 2 / p) * theory_oracle.c2 * p theory_oracle.var = 2 * theory_oracle.c2**2 * p theory_oracle.std = np.sqrt(theory_oracle.var) pp.pprint(theory_oracle) # -- create evaluator for ave; simple -- iterations, K = 1, 1 subsizes = [] block_batchsize = 32 eval_ave_simp = EvalBlockScores(score_fxn_ave, "ave", patchsize, block_batchsize, None) # -- create evaluator for ave -- iterations, K = 1, 1 subsizes = [] eval_ave = EvalBlockScores(score_fxn_ave, "ave", patchsize, block_batchsize, None) # -- create evaluator for bootstrapping -- block_batchsize = 32 eval_prop = EvalBlockScores(score_fxn_bs, "bs", patchsize, block_batchsize, None) # -- init flownet model -- cfg.gpuid = 1 - cfg.gpuid # flip. flop. flownet_align = get_align_method(cfg, "flownet_v2", comp_align=False) cfg.gpuid = 1 - cfg.gpuid # flippity flop. # -- get an image transform -- image_xform = get_image_xform(cfg.image_xform, cfg.gpuid, cfg.frame_size) blockLabels, _ = nnf_utils.getBlockLabels(None, nblocks, np.int32, cfg.device, True) # -- iterate over images -- NUM_BATCHES = min(NUM_BATCHES, len(image_iter)) for image_bindex in range(NUM_BATCHES): print("-=" * 30 + "-") print(f"Running image batch index: {image_bindex}") print("-=" * 30 + "-") torch.cuda.empty_cache() # -- sample & unpack batch -- nwaste = 0 for w in range(nwaste): sample = next(image_iter) # waste one sample = next(image_iter) sample_to_cuda(sample) convert_keys(sample) torch.cuda.synchronize() # for key,val in sample.items(): # print(key,type(val)) # if torch.is_tensor(val): # print(key,val.device) dyn_noisy = sample['dyn_noisy'] # dynamics and noise dyn_clean = sample['dyn_clean'] # dynamics and no noise static_noisy = sample['static_noisy'] # no dynamics and noise static_clean = sample['static_clean'] # no dynamics and no noise nnf_gt = sample['nnf'] flow_gt = sample['flow'] if nnf_gt.ndim == 6: nnf_gt = nnf_gt[:, :, 0] # pick top 1 out of K image_index = sample['image_index'] rng_state = sample['rng_state'] # TODO: anscombe is a type of image transform if not (image_xform is None): dyn_clean_ftrs = image_xform(dyn_clean) dyn_noisy_ftrs = image_xform(dyn_noisy) else: dyn_clean_ftrs = dyn_clean dyn_noisy_ftrs = dyn_noisy if "resize" in cfg.image_xform: vprint("Images, Flows, and NNF Modified.") dyn_clean = image_xform(dyn_clean) dyn_noisy = image_xform(dyn_noisy) T, B, C, H, W = dyn_noisy.shape flow_gt = torch.zeros((B, 1, T, H, W, 2)) nnf_gt = torch.zeros((1, T, B, H, W, 2)) save_image(dyn_clean, "dyn_clean.png") # print("SHAPES") # print(dyn_noisy.shape) # print(dyn_clean.shape) # print(nnf_gt.shape) # -- shape info -- pad = cfg.nblocks // 2 + cfg.patchsize // 2 T, B, C, H, W = dyn_noisy.shape isize = edict({'h': H, 'w': W}) psize = edict({'h': H - 2 * pad, 'w': W - 2 * pad}) ref_t = nframes // 2 nimages, npix, nframes = B, H * W, T frame_size = [H, W] ifsize = [H - 2 * pad, W - 2 * pad] print("flow_gt.shape: ", flow_gt.shape) print("flow_gt: ", flow_gt[0, 0, :, H // 2, W // 2, :]) # -- create results dict -- pixs = edict() flows = edict() anoisy = edict() aligned = edict() runtimes = edict() optimal_scores = edict() # score function at optimal # -- compute proposed search of nnf -- # ave = torch.mean(dyn_noisy_ftrs[:,0,:,4:4+ps,4:4+ps],dim=0) # frames = dyn_noisy_ftrs[:,0,:,4:4+ps,4:4+ps] # gt_offset = torch.sum((frames - ave)**2/nframes).item() # print("Optimal: ",gt_offset) # gt_offset = -1. # -- FIND MODE of BURST -- vprint("Our Method") flow_fmt = rearrange(flow_gt, 'i 1 t h w two -> t i h w 1 two') locs_fmt = flow2locs(flow_fmt) print("locs_fmt.shape: ", locs_fmt.shape) print(dyn_noisy_ftrs.min(), dyn_noisy_ftrs.max()) vals, _ = evalAtLocs(dyn_noisy_ftrs, locs_fmt, patchsize, nblocks, return_mode=False) vals = torch.zeros_like(vals) # flow_fmt = rearrange(flow_gt,'i t h w two -> i (h w) t two') # vals,_ = bnnf_utils.evalAtFlow(dyn_noisy_ftrs, flow_fmt, patchsize, # nblocks, return_mode=False) mode = mode_vals(vals, ifsize) cc_vals = vals[0, 5:29, 5:29, 0].ravel() vstd = torch.std(cc_vals).item() print("[SubBurst] Computed Mode: ", mode) print("[SubBurst] Computed Std: ", vstd) # -- compute proposed search of nnf -- vprint("dyn_noisy_ftrs.shape ", dyn_noisy_ftrs.shape) valMean = theory_npn.mode vprint("valMean: ", valMean) start_time = time.perf_counter() if cfg.nframes < 5: _, flows.est = bnnf_utils.runBurstNnf(dyn_noisy_ftrs, patchsize, nblocks, k=1, valMean=valMean, blockLabels=None, fmt=True, to_flow=True) else: flows.est = rearrange(flow_gt, 'i 1 t h w two -> 1 i (h w) t two').clone() flows.est = flows.est[0] runtimes.est = time.perf_counter() - start_time pixs.est = flow_to_pix(flows.est.clone(), nframes, isize=isize) aligned.est = align_from_flow(dyn_clean, flows.est, patchsize, isize=isize) if cfg.nframes > 7: aligned.est = torch.zeros_like(aligned.est) anoisy.est = align_from_flow(dyn_noisy, flows.est, patchsize, isize=isize) optimal_scores.est = np.zeros((nimages, npix, 1, nframes)) # -- the proposed method -- std = cfg.noise_params.g.std start_time = time.perf_counter() _flow = flow_gt.clone() # _,_flow = runKmSearch(dyn_noisy_ftrs, patchsize, nblocks, k = 1, # std = std/255.,mode="cuda") runtimes.kmb = time.perf_counter() - start_time flows.kmb = rearrange(_flow, 'i 1 t h w two -> i (h w) t two') pixs.kmb = flow_to_pix(flows.kmb.clone(), nframes, isize=isize) aligned.kmb = align_from_flow(dyn_clean, flows.kmb, 0, isize=isize) optimal_scores.kmb = torch_to_numpy(optimal_scores.est) # -- compute proposed search of nnf -- vprint("Our BpSearch Method") # print(flow_gt) # std = cfg.noise_params.g.std/255. valMean = theory_npn.mode start_time = time.perf_counter() bp_nblocks = 3 # _,bp_est,a_noisy = runBpSearch(dyn_noisy_ftrs, dyn_noisy_ftrs, # patchsize, bp_nblocks, k = 1, # valMean = valMean, std=std, # blockLabels=None, # l2_nblocks=nblocks, # fmt = True, to_flow=True, # search_type=cfg.bp_type, # gt_info={'flow':flow_gt}) bp_est = flows.est[None, :].clone() flows.bp_est = bp_est[0] # flows.bp_est = rearrange(flow_gt,'i t h w two -> i (h w) t two') runtimes.bp_est = time.perf_counter() - start_time pixs.bp_est = flow_to_pix(flows.bp_est.clone(), nframes, isize=isize) # aligned.bp_est = a_clean aligned.bp_est = align_from_flow(dyn_clean, flows.bp_est, patchsize, isize=isize) anoisy.bp_est = align_from_flow(dyn_noisy, flows.bp_est, patchsize, isize=isize) optimal_scores.bp_est = np.zeros((nimages, npix, 1, nframes)) # -- compute proposed search of nnf [with tiling ]-- vprint("Our Burst Method (Tiled)") valMean = 0. start_time = time.perf_counter() if cfg.nframes < 5: _, flows.est_tile = bnnf_utils.runBurstNnf(dyn_noisy_ftrs, patchsize, nblocks, k=1, valMean=valMean, blockLabels=None, fmt=True, to_flow=True, tile_burst=True) else: flows.est_tile = rearrange( flow_gt, 'i 1 t h w two -> 1 i (h w) t two').clone() flows.est_tile = flows.est_tile[0] # flows.est_tile = rearrange(flow_gt,'i t h w two -> i (h w) t two') runtimes.est_tile = time.perf_counter() - start_time pixs.est_tile = flow_to_pix(flows.est_tile.clone(), nframes, isize=isize) aligned.est_tile = align_from_flow(dyn_clean, flows.est_tile, patchsize, isize=isize) if cfg.nframes > 7: aligned.est_tile = torch.zeros_like(aligned.est_tile) anoisy.est_tile = align_from_flow(dyn_noisy, flows.est_tile, patchsize, isize=isize) optimal_scores.est_tile = np.zeros((nimages, npix, 1, nframes)) # -- compute new est method -- vprint("[Burst-LK] loss function") vprint(flow_gt.shape) # print(flow_gt[0,:3,32,32,:]) vprint(flow_gt.shape) start_time = time.perf_counter() if frame_size[0] <= 64 and cfg.nblocks < 10 and True: flows.blk = burstNnf.run(dyn_noisy_ftrs, patchsize, nblocks) else: flows.blk = rearrange(flow_gt, 'i 1 t h w two -> i (h w) t two') runtimes.blk = time.perf_counter() - start_time pixs.blk = flow_to_pix(flows.blk.clone(), nframes, isize=isize) aligned.blk = align_from_flow(dyn_clean, flows.blk, patchsize, isize=isize) optimal_scores.blk = np.zeros((nimages, npix, 1, nframes)) # optimal_scores.blk = eval_prop.score_burst_from_flow(dyn_noisy,flows.nnf_local, # patchsize,nblocks)[1] optimal_scores.blk = torch_to_numpy(optimal_scores.blk) # -- compute new est method -- vprint("Oracle") vprint(flow_gt.shape) # print(flow_gt[0,:3,32,32,:]) vprint(flow_gt.shape) valMean = theory_oracle.mode oracle_burst = dyn_noisy_ftrs.clone() oracle_burst[nframes // 2] = dyn_clean_ftrs[nframes // 2] start_time = time.perf_counter() vals_oracle, pix_oracle = nnf_utils.runNnfBurst( oracle_burst, patchsize, nblocks, 1, valMean=valMean, blockLabels=blockLabels) runtimes.oracle = time.perf_counter() - start_time pixs.oracle = rearrange(pix_oracle, 't i h w 1 two -> i (h w) t two') flows.oracle = pix_to_flow(pixs.oracle.clone()) aligned.oracle = align_from_flow(dyn_clean, flows.oracle, patchsize, isize=isize) optimal_scores.oracle = np.zeros((nimages, npix, 1, nframes)) optimal_scores.oracle = torch_to_numpy(optimal_scores.blk) # -- compute optical flow -- vprint("[C Flow]") vprint(dyn_noisy_ftrs.shape) start_time = time.perf_counter() # flows.cflow = cflow.runBurst(dyn_clean_ftrs) # flows.cflow[...,1] = -flows.cflow[...,1] flows.cflow = torch.LongTensor(flows.blk.clone().cpu().numpy()) # flows.cflow = flows.blk.clone() # flows.cflow = torch.round(flows.cflow) runtimes.cflow = time.perf_counter() - start_time pixs.cflow = flow_to_pix(flows.cflow.clone(), nframes, isize=isize) aligned.cflow = align_from_flow(dyn_clean, flows.cflow, patchsize, isize=isize) optimal_scores.cflow = np.zeros((nimages, npix, 1, nframes)) # optimal_scores.blk = eval_prop.score_burst_from_flow(dyn_noisy,flows.nnf_local, # patchsize,nblocks)[1] optimal_scores.blk = torch_to_numpy(optimal_scores.blk) # -- compute groundtruth flow -- dsname = cfg.dataset.name if "kitti" in dsname or 'bsd_burst' == dsname: pix_gt = nnf_gt.type(torch.float) if pix_gt.ndim == 3: pix_gt_rs = rearrange(pix_gt, 'i tm1 two -> i 1 tm1 two') pix_gt = repeat(pix_gt, 'i tm1 two -> i p tm1 two', p=npix) if pix_gt.ndim == 5: pix_gt = rearrange(pix_gt, 't i h w two -> i (h w) t two') pix_gt = torch.LongTensor(pix_gt.cpu().numpy().copy()) # flows.of = torch.zeros_like(pix_gt)#pix_to_flow(pix_gt.clone()) flows.of = pix_to_flow(pix_gt.clone()) else: flows.of = flow_gt flows.of = rearrange(flow_gt, 'i 1 t h w two -> i (h w) t two') # -- align groundtruth flow -- aligned.of = align_from_flow(dyn_clean, flows.of, nblocks, isize=isize) pixs.of = flow_to_pix(flows.of.clone(), nframes, isize=isize) runtimes.of = 0. # given optimal_scores.of = np.zeros( (nimages, npix, 1, nframes)) # clean target is zero aligned.clean = static_clean anoisy.clean = static_clean # optimal_scores.of = eval_ave.score_burst_from_flow(dyn_noisy, # flows.of, # patchsize,nblocks)[0] # -- compute nearest neighbor fields [global] -- vprint("NNF Global.") start_time = time.perf_counter() shape_str = 't b h w two -> b (h w) t two' nnf_vals, nnf_pix = nnf.compute_burst_nnf(dyn_clean_ftrs, ref_t, patchsize) runtimes.nnf = time.perf_counter() - start_time pixs.nnf = torch.LongTensor(rearrange(nnf_pix[..., 0, :], shape_str)) flows.nnf = pix_to_flow(pixs.nnf.clone()) vprint(dyn_clean.shape, pixs.nnf.shape, nblocks) aligned.nnf = align_from_pix(dyn_clean, pixs.nnf, nblocks) anoisy.nnf = align_from_pix(dyn_noisy, pixs.nnf, nblocks) # aligned.nnf = align_from_flow(dyn_clean,flows.nnf,nblocks,isize=isize) optimal_scores.nnf = np.zeros( (nimages, npix, 1, nframes)) # clean target is zero # -- compute nearest neighbor fields [local] -- vprint("NNF Local.") start_time = time.perf_counter() valMean = 0. vals_local, pix_local = nnf_utils.runNnfBurst(dyn_clean_ftrs, patchsize, nblocks, 1, valMean=valMean, blockLabels=blockLabels) runtimes.nnf_local = time.perf_counter() - start_time torch.cuda.synchronize() vprint("pix_local.shape ", pix_local.shape) pixs.nnf_local = rearrange(pix_local, 't i h w 1 two -> i (h w) t two') flows.nnf_local = pix_to_flow(pixs.nnf_local.clone()) # aligned_local = align_from_flow(clean,flow_gt,cfg.nblocks) # aligned_local = align_from_pix(dyn_clean,pix_local,cfg.nblocks) vprint(flows.nnf_local.min(), flows.nnf_local.max()) aligned.nnf_local = align_from_pix(dyn_clean, pixs.nnf_local, nblocks) anoisy.nnf_local = align_from_pix(dyn_noisy, pixs.nnf_local, nblocks) optimal_scores.nnf_local = optimal_scores.nnf # optimal_scores.nnf_local = eval_ave.score_burst_from_flow(dyn_noisy, # flows.nnf, # patchsize,nblocks)[1] optimal_scores.nnf_local = torch_to_numpy(optimal_scores.nnf_local) # ----------------------------------- # # -- old way to compute NNF local -- # # ----------------------------------- # pixs.nnf = torch.LongTensor(rearrange(nnf_pix[...,0,:],shape_str)) # flows.nnf = pix_to_flow(pixs.nnf.clone()) # aligned.nnf = align_from_pix(dyn_clean,pixs.nnf,nblocks) # aligned.nnf = align_from_flow(dyn_clean,flows.nnf,nblocks,isize=isize) # flows.nnf_local = optim.run(dyn_clean_ftrs,patchsize,eval_ave, # nblocks,iterations,subsizes,K) # ----------------------------------- # ----------------------------------- # -- compute proposed search of nnf -- vprint("Global NNF Noisy") start_time = time.perf_counter() split_vals, split_pix = nnf.compute_burst_nnf(dyn_noisy_ftrs, ref_t, patchsize) runtimes.split = time.perf_counter() - start_time # split_pix = np.copy(nnf_pix) split_pix_best = torch.LongTensor( rearrange(split_pix[..., 0, :], shape_str)) split_pix_best = torch.LongTensor(split_pix_best) pixs.split = split_pix_best.clone() flows.split = pix_to_flow(split_pix_best) aligned.split = align_from_pix(dyn_clean, split_pix_best, nblocks) anoisy.split = align_from_pix(dyn_noisy, split_pix_best, nblocks) optimal_scores.split = optimal_scores.nnf_local # optimal_scores.split = eval_ave.score_burst_from_flow(dyn_noisy,flows.nnf_local, # patchsize,nblocks)[1] optimal_scores.split = torch_to_numpy(optimal_scores.split) # -- compute complex ave -- iterations, K = 0, 1 subsizes = [] vprint("[Ours] Ave loss function") start_time = time.perf_counter() estVar = torch.std(dyn_noisy_ftrs.reshape(-1)).item()**2 valMean = 0. #2 * estVar# * patchsize**2# / patchsize**2 vals_local, pix_local = nnf_utils.runNnfBurst(dyn_noisy_ftrs, patchsize, nblocks, 1, valMean=valMean, blockLabels=blockLabels) runtimes.ave = time.perf_counter() - start_time pixs.ave = rearrange(pix_local, 't i h w 1 two -> i (h w) t two') flows.ave = pix_to_flow(pixs.ave.clone()) optimal_scores.ave = optimal_scores.split # same "ave" function aligned.ave = align_from_flow(dyn_clean, flows.ave, nblocks, isize=isize) anoisy.ave = align_from_flow(dyn_noisy, flows.ave, nblocks, isize=isize) optimal_scores.ave = optimal_scores.split # same "ave" function # -- compute ave with smoothing -- iterations, K = 0, 1 subsizes = [] vprint("[Ours] Ave loss function") start_time = time.perf_counter() pix_local = smooth_locs(pix_local, nclusters=1) runtimes.ave_smooth = time.perf_counter() - start_time + runtimes.ave pixs.ave_smooth = rearrange(pix_local, 't i h w 1 two -> i (h w) t two') flows.ave_smooth = pix_to_flow(pixs.ave_smooth.clone()) optimal_scores.ave_smooth = optimal_scores.split # same "ave" function aligned.ave_smooth = align_from_flow(dyn_clean, flows.ave_smooth, nblocks, isize=isize) anoisy.ave_smooth = align_from_flow(dyn_noisy, flows.ave_smooth, nblocks, isize=isize) optimal_scores.ave_smooth = optimal_scores.split # same "ave_smooth" function # -- compute flow -- vprint("L2-Local Recursive") start_time = time.perf_counter() vals_local, pix_local, wburst = nnf_utils.runNnfBurstRecursive( dyn_noisy_ftrs, dyn_clean, patchsize, nblocks, isize, 1, valMean=valMean, blockLabels=blockLabels) runtimes.l2r = time.perf_counter() - start_time pixs.l2r = rearrange(pix_local, 't i h w 1 two -> i (h w) t two') flows.l2r = pix_to_flow(pixs.l2r.clone()) aligned.l2r = wburst #align_from_flow(dyn_clean,flows.l2r,nblocks,isize=isize) optimal_scores.l2r = optimal_scores.split # same "ave" function # -- compute nvof flow -- vprint("NVOF") start_time = time.perf_counter() # flows.nvof = nvof.nvof_burst(dyn_noisy_ftrs) flows.nvof = flows.ave.clone() runtimes.nvof = time.perf_counter() - start_time pixs.nvof = flow_to_pix(flows.nvof.clone(), nframes, isize=isize) aligned.nvof = align_from_flow(dyn_clean, flows.nvof, nblocks, isize=isize) anoisy.nvof = align_from_flow(dyn_noisy, flows.nvof, nblocks, isize=isize) optimal_scores.nvof = optimal_scores.split # same "ave" function # -- compute flownet -- vprint("FlowNetv2") start_time = time.perf_counter() _, flows.flownet = flownet_align(dyn_noisy_ftrs) # flows.flownet = flows.ave.clone().cpu() runtimes.flownet = time.perf_counter() - start_time pixs.flownet = flow_to_pix(flows.flownet.clone(), nframes, isize=isize) aligned.flownet = align_from_flow(dyn_clean, flows.flownet, nblocks, isize=isize) anoisy.flownet = align_from_flow(dyn_noisy, flows.flownet, nblocks, isize=isize) optimal_scores.flownet = optimal_scores.split # -- compute simple ave -- iterations, K = 0, 1 subsizes = [] vprint("[simple] Ave loss function") start_time = time.perf_counter() optim = AlignOptimizer("v3") if cfg.patchsize < 11 and cfg.frame_size[0] <= 64 and False: flows.ave_simp = optim.run(dyn_noisy, patchsize, eval_ave_simp, nblocks, iterations, subsizes, K) else: flows.ave_simp = flows.ave.clone().cpu() runtimes.ave_simp = time.perf_counter() - start_time pixs.ave_simp = flow_to_pix(flows.ave_simp.clone(), nframes, isize=isize) aligned.ave_simp = align_from_flow(dyn_clean, flows.ave_simp, nblocks, isize=isize) anoisy.ave_simp = align_from_flow(dyn_noisy, flows.ave_simp, nblocks, isize=isize) optimal_scores.ave_simp = optimal_scores.split # same "ave" function # -- format results -- #pad = 2*(nframes-1)*ppf+4 # pad = 2*(cfg.nblocks//2)#2*(nframes-1)*ppf+4 # isize = edict({'h':H-pad,'w':W-pad}) # -- flows to numpy -- frame_size = cfg.frame_size[0] is_even = frame_size % 2 == 0 mid_pix = frame_size * frame_size // 2 + (frame_size // 2) * is_even mid_pix = 32 * 10 + 23 flows_np = edict_torch_to_numpy(flows) pixs_np = edict_torch_to_numpy(pixs) # -- End-Point-Errors -- epes_of = compute_flows_epe_wrt_ref(flows, "of") epes_nnf = compute_flows_epe_wrt_ref(flows, "nnf") epes_nnf_local = compute_flows_epe_wrt_ref(flows, "nnf_local") nnf_acc = compute_acc_wrt_ref(flows, "nnf") nnf_local_acc = compute_acc_wrt_ref(flows, "nnf_local") # -- PSNRs -- aligned = remove_center_frames(aligned) psnrs = compute_frames_psnr(aligned, psize) # -- denoised PSNRS -- def burst_mean(in_burst): return torch.mean(in_burst, dim=0)[None, :] anoisy = remove_center_frames(anoisy) anoisy = apply_across_dict(anoisy, burst_mean) dn_psnrs = compute_frames_psnr(anoisy, psize) vprint(dn_psnrs) # -- print report --- print("\n" * 3) # banner print("-" * 25 + " Results " + "-" * 25) # print_dict_ndarray_0_midpix(flows_np,mid_pix) # print_dict_ndarray_0_midpix(pixs_np,mid_pix) # print_verbose_psnrs(psnrs) # print_delta_summary_psnrs(psnrs) # print_verbose_epes(epes_of,epes_nnf) # print_nnf_acc(nnf_acc) # print_nnf_local_acc(nnf_local_acc) # print_summary_epes(epes_of,epes_nnf) # print_summary_denoised_psnrs(dn_psnrs) print_summary_psnrs(psnrs) print_runtimes(runtimes) # -- prepare results to be appended -- psnrs = edict_torch_to_numpy(psnrs) epes_of = edict_torch_to_numpy(epes_of) epes_nnf = edict_torch_to_numpy(epes_nnf) epes_nnf_local = edict_torch_to_numpy(epes_nnf_local) nnf_acc = edict_torch_to_numpy(nnf_acc) nnf_local_acc = edict_torch_to_numpy(nnf_local_acc) image_index = torch_to_numpy(image_index) batch_results = { 'runtimes': runtimes, 'optimal_scores': optimal_scores, 'psnrs': psnrs, 'epes_of': epes_of, 'epes_nnf': epes_nnf, 'epes_nnf_local': epes_nnf_local, 'nnf_acc': nnf_acc, 'nnf_local_acc': nnf_local_acc } # -- format results -- batch_results = flatten_internal_dict(batch_results) format_fields(batch_results, image_index, rng_state) # print("shape check.") # for key,value in batch_results.items(): # print(key,value.shape) record.append(batch_results) # print("\n"*3) # print("-"*20) # print(record.record) # print("-"*20) # print("\n"*3) # record.stack_record() record.cat_record() # print("\n"*3) # print("-"*20) # print(record.record) # print("-"*20) print("\n" * 3) print("-" * 20) # df = pd.DataFrame().append(record.record,ignore_index=True) for key, val in record.record.items(): vprint(key, val.shape) # vprint(df) vprint("-" * 20) vprint("\n" * 3) return record.record
def run_with_seed(seed): # -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # # Settings # # -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # -- settings -- cfg = get_cfg_defaults() cfg.use_anscombe = False cfg.noise_params.ntype = 'g' cfg.noise_params.g.std = 25. cfg.nframes = 3 cfg.dynamic_info.nframes = cfg.nframes cfg.nblocks = 3 cfg.patchsize = 11 cfg.gpuid = 1 cfg.device = f"cuda:{cfg.gpuid}" # -- seeds -- cfg.seed = seed # cfg.seed = 123 # sky of a forest # cfg.seed = 345 # handrail and stairs # cfg.seed = 567 # cloudy blue sky # cfg.seed = 567 # cloudy blue sky # -- set seed -- set_seed(cfg.seed) # -- load dataset -- data, loaders = load_image_dataset(cfg) train_iter = iter(loaders.tr) # -- fetch sample -- sample = next(train_iter) sample_to_cuda(sample) # -- unpack data -- noisy, clean = sample['noisy'], sample['burst'] nframes, nimages, ncolors, H, W = noisy.shape isize = edict({'h': H, 'w': W}) # -- setup results -- scores = edict() scores.noisy = edict() scores.clean = edict() blocks = edict() blocks.noisy = edict() blocks.clean = edict() # -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # # Setup For Searches # # -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # -- tile image to patches -- pad = 2 * (cfg.nblocks // 2) h, w = cfg.patchsize + pad, cfg.patchsize + pad noisy_patches = tile_patches(noisy, cfg.patchsize + pad).pix noisy_patches = rearrange(noisy_patches, 'b t s (h w c) -> b s t c h w', h=h, w=w) nimages, npix, nframes, c, psH, psW = noisy_patches.shape clean_patches = tile_patches(clean, cfg.patchsize + pad).pix clean_patches = rearrange(clean_patches, 'b t s (h w c) -> b s t c h w', h=h, w=w) nimages, npix, nframes, c, psH, psW = clean_patches.shape masks = torch.ones(nimages, npix, nframes, c, psH, psW).to(cfg.device) # -- create constants -- frames = np.r_[np.arange(cfg.nframes // 2), np.arange(cfg.nframes // 2 + 1, cfg.nframes)] frames = repeat(frames, 'z -> i s z', i=nimages, s=npix) brange = exh_block_range(nimages, npix, cfg.nframes, cfg.nblocks) curr_blocks = init_optim_block(nimages, npix, cfg.nframes, cfg.nblocks) srch_blocks = get_search_blocks(frames, brange, curr_blocks, cfg.device) np_srch_blocks = torch_to_numpy(srch_blocks[0]) S = len(srch_blocks[0, 0]) # -- create constants -- frames_pair = np.array([0]) frames = repeat(frames_pair, 'z -> i s z', i=nimages, s=npix) brange = exh_block_range(nimages, npix, cfg.nframes, cfg.nblocks) curr_blocks_pair = init_optim_block(nimages, npix, cfg.nframes, cfg.nblocks) srch_blocks_pair = get_search_blocks(frames, brange, curr_blocks_pair, cfg.device) S_pair = len(srch_blocks[0, 0]) # -- encode blocks -- single_search_block = srch_blocks[0, 0].cpu().numpy() block_strings = search_blocks_to_str(single_search_block) labels = search_blocks_to_labels(single_search_block, block_strings) # -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # # Execute Searches # # -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # # --- run PAIRED split search --- # ave_fxn = get_score_function("ave") block_batchsize = 128 evaluator = EvalBlockScores(ave_fxn, "ave", cfg.patchsize, block_batchsize, None) get_topK = evaluator.compute_topK_scores # -- a) run clean -- clean_scores, clean_blocks = get_topK(clean_patches, masks, srch_blocks_pair, cfg.nblocks, S_pair) scores_full = torch_to_numpy(clean_scores[0]) blocks_full = torch_to_numpy(clean_blocks[0]) # -- b) tile results to full blocks -- scores_full, blocks_full = tile_pair_to_full(scores_full, blocks_full, np_srch_blocks, frames_pair, cfg.nframes, cfg.nblocks) scores.clean.ave = scores_full blocks.clean.ave = batch_search_blocks_to_labels(blocks_full, block_strings) # -- a) run noisy -- noisy_scores, noisy_blocks = get_topK(noisy_patches, masks, srch_blocks_pair, cfg.nblocks, S_pair) scores_full = torch_to_numpy(noisy_scores[0]) blocks_full = torch_to_numpy(noisy_blocks[0]) # -- b) tile results to full blocks -- scores_full, blocks_full = tile_pair_to_full(scores_full, blocks_full, np_srch_blocks, frames_pair, cfg.nframes, cfg.nblocks) scores.noisy.ave = scores_full blocks.noisy.ave = batch_search_blocks_to_labels(blocks_full, block_strings) # # --- run FULL split search --- # ave_fxn = get_score_function("ave") block_batchsize = 128 evaluator = EvalBlockScores(ave_fxn, "ave", cfg.patchsize, block_batchsize, None) get_topK = evaluator.compute_topK_scores # -- run clean -- clean_scores, clean_blocks = get_topK(clean_patches, masks, srch_blocks, cfg.nblocks, S) clean_scores = torch_to_numpy(clean_scores) scores.clean.full_ave = clean_scores[0] clean_blocks = torch_to_numpy(clean_blocks) batch_blocks = clean_blocks[0, :, :, :] blocks.clean.full_ave = batch_search_blocks_to_labels( batch_blocks, block_strings) # -- run noisy -- noisy_scores, noisy_blocks = get_topK(noisy_patches, masks, srch_blocks, cfg.nblocks, S) noisy_scores = torch_to_numpy(noisy_scores) scores.noisy.full_ave = noisy_scores[0] noisy_blocks = torch_to_numpy(noisy_blocks) batch_blocks = noisy_blocks[0, :, :, :] blocks.noisy.full_ave = batch_search_blocks_to_labels( batch_blocks, block_strings) # # --- run bootstrapping --- # bs_fxn = get_score_function("bootstrapping_mod2") block_batchsize = 32 evaluator = EvalBlockScores(bs_fxn, "bs_mod2", cfg.patchsize, block_batchsize, None) get_topK = evaluator.compute_topK_scores # -- run noisy -- noisy_scores, noisy_blocks = get_topK(noisy_patches, masks, srch_blocks, cfg.nblocks, S) noisy_scores = torch_to_numpy(noisy_scores) scores.noisy.bs = noisy_scores[0] noisy_blocks = torch_to_numpy(noisy_blocks) batch_blocks = noisy_blocks[0, :, :, :] blocks.noisy.bs = batch_search_blocks_to_labels(batch_blocks, block_strings) # -- run clean -- clean_scores, clean_blocks = get_topK(clean_patches, masks, srch_blocks, cfg.nblocks, S) clean_scores = torch_to_numpy(clean_scores) scores.clean.bs = clean_scores[0] clean_blocks = torch_to_numpy(clean_blocks) batch_blocks = noisy_blocks[0, :, :, :] blocks.clean.bs = batch_search_blocks_to_labels(batch_blocks, block_strings) # -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- # # Plot Results # # -=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- print("Plotting Results.") plot_landscape(scores, blocks, seed)