def regularized_fine(lenses,
fine_costs,
disp,
penalty1,
penalty2,
max_cost,
conf_sigma=0.3,
min_thresh=2.0,
eps=0.0000001):
fine_depths = dict()
fine_depths_interp = dict()
fine_depths_val = dict()
wta_depths = dict()
wta_depths_interp = dict()
wta_depths_val = dict()
confidence = dict()
for i, l in enumerate(fine_costs):
if i % 1000 == 0:
print("Processing lens {0}".format(i))
lens = lenses[l]
# prepare the cost shape: disparity axis is third axis (index [2] instead of [0])
F = np.flipud(np.rot90(fine_costs[l].T))
# the regularized cost volume
sgm_cost = rtxsgm.sgm(lenses[l].img, F, lens.mask, penalty1, penalty2,
False, max_cost)
# plain minima
fine_depths[l] = np.argmin(sgm_cost, axis=2)
# interpolated minima and values
fine_depths_interp[l], fine_depths_val[l] = rtxdisp.cost_minima_interp(
sgm_cost, disp)
if i % 1000 == 0:
print("max interp: {0}".format(np.amax(fine_depths_interp[l])))
# confidence measure used in "Real-Time Visibility-Based Fusion of Depth Maps"
# substract 1 at the end since the "real" optimium is included in the vectorized operations
#confidence[l][fine_depths_val[l] > min_thresh] = 0.0
# plain winner takes all minima
wta_depths[l] = np.argmin(F, axis=2)
# interpolated minima and values from the unregularized cost volume
wta_depths_interp[l], wta_depths_val[l] = rtxdisp.cost_minima_interp(
F, disp)
confidence[l] = np.sum(np.exp(-(
(sgm_cost - fine_depths_val[l][:, :, None])**2) / conf_sigma),
axis=2) - 1
#TODO: calculate wta confidence, scale the sigma accordingly
#confidence[l] = np.sum(np.exp(-((F - wta_depths_val[l][:, :, None])**2) / conf_sigma), axis=2) - 1
# avoid overflow in division
ind = confidence[l] > eps
confidence[l][confidence[l] <= 0] = 0.0
confidence[l][ind] = 1.0 / confidence[l][ind]
return fine_depths, fine_depths_interp, fine_depths_val, wta_depths, wta_depths_interp, wta_depths_val, confidence
def regularized_fine(lenses, fine_costs, disp, penalty1, penalty2, max_cost, conf_tec='mlm', conf_sigma=0.3, min_thresh=2.0, eps=0.0000001):
fine_depths = dict()
fine_depths_interp = dict()
fine_depths_val = dict()
wta_depths = dict()
wta_depths_interp = dict()
wta_depths_val = dict()
num_lenses = len(lenses)
confidence = dict()
for i, l in enumerate(fine_costs):
if i%1000==0:
print("Regularization: Processing lens {0}/{1}".format(i, num_lenses))
lens = lenses[l]
# prepare the cost shape: disparity axis is third axis (index [2] instead of [0])
F = np.flipud(np.rot90(fine_costs[l].T))
# the regularized cost volume
sgm_cost = rtxsgm.sgm(lenses[l].img, F, lens.mask, penalty1, penalty2, False, max_cost)
# plain minima
fine_depths[l] = np.argmin(sgm_cost, axis=2)
# interpolated minima and values
#fine_depths_interp[l], fine_depths_val[l] = rtxdisp.cost_minima_interp(sgm_cost, disp)
# cost should be C-Contiguous
sgm_cost_c = sgm_cost.copy(order='C')
# also in int32
sgm_cost_c_int = (sgm_cost_c * 1000).astype(np.int32)
# number of disparities
n_disps = F.shape[2]
cut = 'unary'
if cut == 'potts':
# potts model
depth_cut = cut_simple(sgm_cost_c_int, -5 * np.eye(sgm_cost.shape[2], dtype=np.int32))
elif cut == 'unary':
# unary model
x, y = np.ogrid[:n_disps, :n_disps]
one_d_topology = np.abs(x - y).astype(np.int32).copy("C")
#pdb.set_trace()
depth_cut = cut_simple(sgm_cost_c_int, 5 * one_d_topology)
else:
print("No cut recognised. Do you wish to use potts or unary model?")
pdb.set_trace()
fine_depths_interp[l] = depth_cut
#if i%1000==0:
# print("max interp: {0}".format(np.amax(fine_depths_interp[l])))
# plain winner takes all minima
wta_depths[l] = np.argmin(F, axis=2)
# interpolated minima and values from the unregularized cost volume
wta_depths_interp[l], wta_depths_val[l] = rtxdisp.cost_minima_interp(F, disp)
fine_depths_val[l] = wta_depths_val[l]
### CALCULATE THE CONFIDENCE USING A METHOD
minimum_costs = np.min(sgm_cost, axis=2)
#pdb.set_trace()
if conf_tec == 'oev':
# TODO max does not work on arrays
num_denom = 0
dmax = np.max(sgm_cost)
dmin = np.min(sgm_cost)
denom_denom = max(sgm_cost - minimum_costs[:,:,None], 1)
for n in range(0, sgm_cost.shape[2]):
index_map = np.ones((sgm_cost.shape[0], sgm_cost.shape[1])) * n
tmp_num = np.pow(max(min(index_map - fine_depths[l], (dmax - dmin)/3), 0), 2)
num_denom += tmp_num / denom_denom[:,:,n]
confidence[l] = 1 / num_denom
elif conf_tec == 'rtvbf':
confidence[l] = np.sum(np.exp(-((sgm_cost - fine_depths_val[l][:, :, None])**2) / conf_sigma), axis=2) - 1
# confidence measure used in "Real-Time Visibility-Based Fusion of Depth Maps"
# substract 1 at the end since the "real" optimium is included in the vectorized operations
# confidence[l][fine_depths_val[l] > min_thresh] = 0.0
#TODO: calculate wta confidence, scale the sigma accordingly
#confidence[l] = np.sum(np.exp(-((F - wta_depths_val[l][:, :, None])**2) / conf_sigma), axis=2) - 1
# avoid overflow in division
ind = confidence[l] > eps
confidence[l][confidence[l] <= 0] = 0.0
confidence[l][ind] = 1.0 / confidence[l][ind]
else:
# TODO
# check overflow in exp
# and division for zero
#conf_tec == 'mlm':
exp_cost = np.exp(-minimum_costs/(2*np.power(conf_sigma,2)))
denom_cost = np.sum(np.exp(-sgm_cost/(2*np.power(conf_sigma,2))), axis=2)
#zeros = denom_cost == 0
#denom_cost = denom_cost + zeros * np.max(denom_cost)
confidence_map = exp_cost / denom_cost
confidence_map[np.isnan(confidence_map)] = 0
confidence[l] = confidence_map
return fine_depths, fine_depths_interp, fine_depths_val, wta_depths, wta_depths_interp, wta_depths_val, confidence
view, filt_size)
cost_volume[:, :,
i] = filter_cost_volume_slice(cost_volume_slice,
args.cv_filtering,
median_filter_kernel_size)
#pdb.set_trace()
# COST REFINEMENT
if args.sgm:
print("Applying semi-global matching to refine cost volume..")
F = np.flipud(np.rot90(cost_volume.T))
#pdb.set_trace()
# the regularized cost volume
imgI = filt.rgb2gray(view)
mask = np.ones_like(imgI)
cost_volume = rtxsgm.sgm(imgI, cost_volume, mask, penalty1, penalty2,
False, max_cost)
# CONFIDENCE
confidence_map = est_conf_map(cost_volume, conf_sigma)
good_pixels_map = confidence_map > (np.mean(confidence_map) -
std_allow * np.std(confidence_map))
# DISPARITY EXTRACTION
print("Extracting disparity..")
# here we want to allineate disparities, so we use the patch size as common factor
# RENDERING FROM DISP: ps = disp * info['dmax']
# RENDERING FOCAL STACK: ps = fp * calib.lens_diameter / 2
# ps = ps --> disp = (fp * calib.lens_diameter) / (2 * info['dmax'])
# there is a 0.5 factor somewhere that I lost
# calib is calibs[0]
def regularized_fine(lenses, fine_costs, disp, penalty1, penalty2, max_cost, conf_tec='mlm', conf_sigma=0.3, min_thresh=2.0, eps=0.0000001):
fine_depths = dict()
fine_depths_interp = dict()
fine_depths_val = dict()
wta_depths = dict()
wta_depths_interp = dict()
wta_depths_val = dict()
num_lenses = len(lenses)
confidence = dict()
for i, l in enumerate(fine_costs):
if i%100==0:
print("Regularization: Processing lens {:05d}/{:05d}".format(i, num_lenses), end="\r", flush=True)
lens = lenses[l]
# prepare the cost shape: disparity axis is third axis (index [2] instead of [0])
F = np.flipud(np.rot90(fine_costs[l].T))
# the regularized cost volume
sgm_cost = rtxsgm.sgm(lenses[l].img, F, lens.mask, penalty1, penalty2, False, max_cost)
# plain minima
fine_depths[l] = np.argmin(sgm_cost, axis=2)
# interpolated minima and values
fine_depths_interp[l], fine_depths_val[l] = rtxdisp.cost_minima_interp(sgm_cost, disp)
#if i%1000==0:
# print("max interp: {0}".format(np.amax(fine_depths_interp[l])))
# plain winner takes all minima
wta_depths[l] = np.argmin(F, axis=2)
# interpolated minima and values from the unregularized cost volume
wta_depths_interp[l], wta_depths_val[l] = rtxdisp.cost_minima_interp(F, disp)
fine_depths_val[l] = wta_depths_val[l]
### CALCULATE THE CONFIDENCE USING A METHOD
minimum_costs = np.min(sgm_cost, axis=2)
#pdb.set_trace()
if conf_tec == 'oev':
# TODO max does not work on arrays
num_denom = 0
dmax = np.max(sgm_cost)
dmin = np.min(sgm_cost)
denom_denom = max(sgm_cost - minimum_costs[:,:,None], 1)
for n in range(0, sgm_cost.shape[2]):
index_map = np.ones((sgm_cost.shape[0], sgm_cost.shape[1])) * n
tmp_num = np.pow(max(min(index_map - fine_depths[l], (dmax - dmin)/3), 0), 2)
num_denom += tmp_num / denom_denom[:,:,n]
confidence[l] = 1 / num_denom
elif conf_tec == 'rtvbf':
confidence[l] = np.sum(np.exp(-((sgm_cost - fine_depths_val[l][:, :, None])**2) / conf_sigma), axis=2) - 1
# confidence measure used in "Real-Time Visibility-Based Fusion of Depth Maps"
# substract 1 at the end since the "real" optimium is included in the vectorized operations
# confidence[l][fine_depths_val[l] > min_thresh] = 0.0
#TODO: calculate wta confidence, scale the sigma accordingly
#confidence[l] = np.sum(np.exp(-((F - wta_depths_val[l][:, :, None])**2) / conf_sigma), axis=2) - 1
# avoid overflow in division
ind = confidence[l] > eps
confidence[l][confidence[l] <= 0] = 0.0
confidence[l][ind] = 1.0 / confidence[l][ind]
else:
# TODO
# check overflow in exp
# and division for zero
#conf_tec == 'mlm':
exp_cost = np.exp(-minimum_costs/(2*np.power(conf_sigma,2)))
denom_cost = np.sum(np.exp(-sgm_cost/(2*np.power(conf_sigma,2))), axis=2)
#zeros = denom_cost == 0
#denom_cost = denom_cost + zeros * np.max(denom_cost)
confidence_map = exp_cost / denom_cost
confidence_map[np.isnan(confidence_map)] = 0
confidence[l] = confidence_map
print("\nDone!")
return fine_depths, fine_depths_interp, fine_depths_val, wta_depths, wta_depths_interp, wta_depths_val, confidence