def disparity_minus_mean():
down_factor = 2
# index = 1
test = False
n_disp = 128
differences = []
border = 10
for index in range(2):
print(index)
source = KittiMultiViewSource(index, test=test, n_frames=0)
average_disparity = source.get_average_disparity()
# frame = load_pair(index, test, frame=i, multiview=True)
frame = source.frame_ten
gt_frame = calc_ground_truth(frame,
n_disp,
down_factor=down_factor,
iters=50)
d = gt_frame.astype('int') - average_disparity.astype('int')
d = d[border:-border, border:-border]
differences.append(d.flatten())
# plt.subplot(211)
# plt.imshow(gt_frame)
# plt.colorbar()
# plt.subplot(212)
# plt.imshow(average_disparity)
# plt.colorbar()
# plt.show()
foo = np.array(differences)
print()
print(np.array(differences).shape)
def disparity_minus_mean():
down_factor = 2
# index = 1
test=False
n_disp = 128
differences = []
border = 10
for index in range(2):
print(index)
source = KittiMultiViewSource(index, test=test, n_frames=0)
average_disparity = source.get_average_disparity()
# frame = load_pair(index, test, frame=i, multiview=True)
frame = source.frame_ten
gt_frame = calc_ground_truth(frame, n_disp, down_factor=down_factor, iters=50)
d = gt_frame.astype('int') - average_disparity.astype('int')
d = d[border:-border,border:-border]
differences.append(d.flatten())
# plt.subplot(211)
# plt.imshow(gt_frame)
# plt.colorbar()
# plt.subplot(212)
# plt.imshow(average_disparity)
# plt.colorbar()
# plt.show()
foo = np.array(differences)
print()
print(np.array(differences).shape)
def foveation_sequence():
frame_down_factor = 1
mem_down_factor = 2 # relative to the frame down factor
coarse_down_factor = 2 # for the coarse comparison
fs = 80
fovea_shape = (fs, fs)
full_values = 128
values = full_values / 2**frame_down_factor
index = 15
n_frames = 10
source = KittiMultiViewSource(index, test=False, n_frames=n_frames)
full_shape = source.frame_ten[0].shape
frame_ten = [downsample(source.frame_ten[0], frame_down_factor),
downsample(source.frame_ten[1], frame_down_factor)]
frame_shape = frame_ten[0].shape
average_disp = source.get_average_disparity()
average_disp = cv2.pyrUp(average_disp)[:frame_shape[0],:frame_shape[1]-values]
filter = Filter(average_disp, frame_down_factor, mem_down_factor,
fovea_shape, frame_shape, values, verbose=False, memory_length=0)
plt.figure()
import matplotlib.cm as cm
for i in range(0, 10, 2):
frame = [downsample(source.frame_sequence[i][0], frame_down_factor),
downsample(source.frame_sequence[i][1], frame_down_factor)]
filter_disp, fovea_corner = filter.process_frame(None, frame)
edge = 5
plt.subplot(5,1,i/2+1)
# plt.subplot(5,2,i+1)
plt.imshow(trim(frame[0], values, edge), cmap = cm.Greys_r)
# remove_axes()
# plt.subplot(5,2,i+2)
# plt.imshow(trim(filter_disp, values, edge), vmin=0, vmax=full_values)
fovea_corner = fovea_corner[0]
# plot_edges(fovea_ij, (fs, fs))
fi, fj = fovea_corner
fm = fs
fn = fs
plt.plot([fj, fj+fn, fj+fn, fj, fj], [fi, fi, fi+fm, fi+fm, fi], 'white')
# plt.scatter(fovea_corner[1]-values+fs/2, fovea_corner[0]-edge+fs/2, s=100, c='green', marker='+', linewidths=2)
# plt.scatter(fovea_corner[1]-values, fovea_corner[0]-edge, s=9, c='green', marker='+', linewidths=3)
# plt.scatter(fovea_corner[1]-values+fs, fovea_corner[0]-edge+fs, s=9, c='green', marker='+', linewidths=3)
# plt.scatter(fovea_corner[1]-values, fovea_corner[0]-edge+fs, s=9, c='green', marker='+', linewidths=3)
# plt.scatter(fovea_corner[1]-values+fs, fovea_corner[0]-edge, s=9, c='green', marker='+', linewidths=3)
remove_axes()
plt.tight_layout(-1)
plt.show()
def objective(args):
# args['ksize'] = int(args['ksize'])
costs = []
# for index in range(3):
# for index in range(10):
for index in range(194):
if index in [31, 82, 114]: # missing frames
continue
source = KittiMultiViewSource(index, test=False, n_frames=0)
full_shape = source.frame_ten[0].shape
frame_ten = [downsample(source.frame_ten[0], frame_down_factor),
downsample(source.frame_ten[1], frame_down_factor)]
frame_shape = frame_ten[0].shape
true_points = source.get_ground_truth_points(occluded=False)
# determine fovea shape
fovea_pixels = np.ceil(fovea_fraction * frame_shape[0] * (frame_shape[1] - values))
if fovea_pixels > 0:
fovea_height = np.minimum(frame_shape[0], np.ceil(fovea_pixels ** .5))
fovea_width = np.minimum(frame_shape[1], np.ceil(fovea_pixels / fovea_height))
fovea_shape = fovea_height, fovea_width
else:
fovea_shape = (0, 0)
# average disparity
try:
average_disp = source.get_average_disparity()
except IOError: # likely does not have a full 20 frames
print("Skipping index %d (lacks frames)" % index)
continue
average_disp = upsample_average_disp(
average_disp, frame_down_factor, frame_shape, values)
# run the filter
foveal = Foveal(average_disp, frame_down_factor, mem_down_factor,
fovea_shape, frame_shape, values,
iters=iters, fovea_levels=fovea_levels, **args)
disp, _ = foveal.process_frame(frame_ten)
# cost = cost_on_points(disp[:,values:], true_points, full_shape=full_shape)
cost = cost_on_points(disp[:,values:], true_points, average_disp,
full_shape=full_shape, clip=error_clip)
costs.append(cost)
error = np.mean(costs)
errors[arg_key(args)] = error
args_s = "{%s}" % ', '.join("%r: %s" % (k, args[k]) for k in sorted(space))
print("%s: %s" % (error, args_s))
return error
#plt.tight_layout()
#plt.show(block=True)
area = 93 * 275
fovea_area = 0.4 * area
height = np.floor(np.sqrt(fovea_area))
width = np.floor(fovea_area / height)
fovea_shape = (height, width)
print(np.floor(np.sqrt(area)))
n_frames = 194
best_nums = []
for index in range(n_frames):
print(index)
source = KittiMultiViewSource(index)
average_disparity = source.get_average_disparity()
uc = UnusuallyClose(average_disparity)
gt = calc_ground_truth(source.frame_ten, 128, down_factor=2, iters=3)
cost = uc.get_importance(gt)
foveas_ij, fovea_part_shape = _choose_foveas(cost, fovea_shape, values, 30)
best_nums.append(len(foveas_ij))
print('best nums:')
print(best_nums)
plt.figure()
plt.hist(best_nums, np.add(range(31), 0.5))
plt.xlabel('Number of foveas', fontsize=18)
plt.ylabel('Frequency of best cost', fontsize=18)
plt.gca().tick_params(labelsize='18')
#plt.show(block=True)
area = 93*275
fovea_area = 0.4 * area
height = np.floor(np.sqrt(fovea_area))
width = np.floor(fovea_area / height)
fovea_shape = (height, width)
print(np.floor(np.sqrt(area)))
n_frames = 194
best_nums = []
for index in range(n_frames):
print(index)
source = KittiMultiViewSource(index)
average_disparity = source.get_average_disparity()
uc = UnusuallyClose(average_disparity)
gt = calc_ground_truth(source.frame_ten, 128, down_factor=2, iters=3)
cost = uc.get_importance(gt)
foveas_ij, fovea_part_shape = _choose_foveas(cost, fovea_shape, values, 30)
best_nums.append(len(foveas_ij))
print('best nums:')
print(best_nums)
plt.figure()
plt.hist(best_nums, np.add(range(31), 0.5))
plt.xlabel('Number of foveas', fontsize=18)
plt.ylabel('Frequency of best cost', fontsize=18)
plt.gca().tick_params(labelsize='18')
# --- setup
iters = 3
full_values = 128
frame_down_factor = 1
assert full_values % 2**frame_down_factor == 0
values = full_values / 2**frame_down_factor
sources = []
# for index in range(1):
# for index in range(30):
for index in range(194):
source = KittiMultiViewSource(index, test=False)
frame_ten = [downsample(source.frame_ten[0], frame_down_factor),
downsample(source.frame_ten[1], frame_down_factor)]
true_points = source.get_ground_truth_points(occluded=False)
sources.append((source, frame_ten, true_points))
r = np.log(10)
space = collections.OrderedDict([
('laplacian_ksize', pyll.scope.int(1 + hp.quniform('laplacian_ksize', 0, 20, 2))),
('laplacian_scale', hp.lognormal('laplacian_scale', 0, r)),
('data_weight', hp.lognormal('data_weight', -2*r, r)),
('data_max', hp.lognormal('data_max', 2*r, r)),
('data_exp', hp.lognormal('data_exp', 0, r)),
('disc_max', hp.lognormal('disc_max', r, r)),
def rationale():
"""
Figure that illustrates rationale for fovea approach, i.e. diminishing returns
with increasing runtime via further iterations at a given downfactor, but
possibly too-high cost of using lower downfactor.
"""
# n_frames = 194
n_frames = 50
frame_down_factor = 1
source = KittiMultiViewSource(0)
full_shape = source.frame_ten[0].shape
frame_shape = downsample(source.frame_ten[0], frame_down_factor).shape
params = {'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425, 'data_max': 32.024780646200725, 'laplacian_ksize': 3}
# iterations = [1, 2, 3]
iterations = [1, 2, 3, 4, 5, 7, 10, 15]
mean_coarse_times = []
mean_coarse_costs = []
mean_fine_times = []
mean_fine_costs = []
for j in range(len(iterations)):
print(str(iterations[j]) + ' iterations')
coarse_times = []
coarse_costs = []
fine_times = []
fine_costs = []
for i in range(n_frames):
print(' frame ' + str(i))
source = KittiMultiViewSource(i)
true_points = source.get_ground_truth_points(occluded=False)
frame_down_factor = 1
coarse_time, coarse_cost = _evaluate_frame(source.frame_ten, true_points, frame_shape,
full_shape, frame_down_factor+1, frame_down_factor, iterations[j], params)
fine_time, fine_cost = _evaluate_frame(source.frame_ten, true_points, frame_shape,
full_shape, frame_down_factor+0, frame_down_factor, iterations[j], params)
coarse_times.append(coarse_time)
coarse_costs.append(coarse_cost)
fine_times.append(fine_time)
fine_costs.append(fine_cost)
mean_coarse_times.append(np.mean(coarse_times))
mean_coarse_costs.append(np.mean(coarse_costs))
mean_fine_times.append(np.mean(fine_times))
mean_fine_costs.append(np.mean(fine_costs))
print(mean_coarse_times)
print(mean_coarse_costs)
print(mean_fine_times)
print(mean_fine_costs)
plt.plot(mean_coarse_times, mean_coarse_costs, color='k', marker='s', markersize=12)
plt.plot(mean_fine_times, mean_fine_costs, color='k', marker='o', markersize=12)
plt.xlabel('Runtime (s)', fontsize=18)
plt.ylabel('Mean absolute disparity error (pixels)', fontsize=18)
plt.gca().tick_params(labelsize='18')
plt.show()
from filter import cost_on_points, expand_coarse
# --- setup
iters = 3
full_values = 128
frame_down_factor = 1
assert full_values % 2**frame_down_factor == 0
values = full_values / 2**frame_down_factor
sources = []
# for index in range(1):
# for index in range(30):
for index in range(194):
source = KittiMultiViewSource(index, test=False)
frame_ten = [
downsample(source.frame_ten[0], frame_down_factor),
downsample(source.frame_ten[1], frame_down_factor)
]
true_points = source.get_ground_truth_points(occluded=False)
sources.append((source, frame_ten, true_points))
r = np.log(10)
space = collections.OrderedDict([
('laplacian_ksize',
pyll.scope.int(1 + hp.quniform('laplacian_ksize', 0, 20, 2))),
('laplacian_scale', hp.lognormal('laplacian_scale', 0, r)),
('data_weight', hp.lognormal('data_weight', -2 * r, r)),
('data_max', hp.lognormal('data_max', 2 * r, r)),
def foveation_sequence():
frame_down_factor = 1
mem_down_factor = 2 # relative to the frame down factor
coarse_down_factor = 2 # for the coarse comparison
fs = 80
fovea_shape = (fs, fs)
full_values = 128
values = full_values / 2**frame_down_factor
index = 15
n_frames = 10
source = KittiMultiViewSource(index, test=False, n_frames=n_frames)
full_shape = source.frame_ten[0].shape
frame_ten = [
downsample(source.frame_ten[0], frame_down_factor),
downsample(source.frame_ten[1], frame_down_factor)
]
frame_shape = frame_ten[0].shape
average_disp = source.get_average_disparity()
average_disp = cv2.pyrUp(average_disp)[:frame_shape[0], :frame_shape[1] -
values]
filter = Filter(average_disp,
frame_down_factor,
mem_down_factor,
fovea_shape,
frame_shape,
values,
verbose=False,
memory_length=0)
plt.figure()
import matplotlib.cm as cm
for i in range(0, 10, 2):
frame = [
downsample(source.frame_sequence[i][0], frame_down_factor),
downsample(source.frame_sequence[i][1], frame_down_factor)
]
filter_disp, fovea_corner = filter.process_frame(None, frame)
edge = 5
plt.subplot(5, 1, i / 2 + 1)
# plt.subplot(5,2,i+1)
plt.imshow(trim(frame[0], values, edge), cmap=cm.Greys_r)
# remove_axes()
# plt.subplot(5,2,i+2)
# plt.imshow(trim(filter_disp, values, edge), vmin=0, vmax=full_values)
fovea_corner = fovea_corner[0]
# plot_edges(fovea_ij, (fs, fs))
fi, fj = fovea_corner
fm = fs
fn = fs
plt.plot([fj, fj + fn, fj + fn, fj, fj],
[fi, fi, fi + fm, fi + fm, fi], 'white')
# plt.scatter(fovea_corner[1]-values+fs/2, fovea_corner[0]-edge+fs/2, s=100, c='green', marker='+', linewidths=2)
# plt.scatter(fovea_corner[1]-values, fovea_corner[0]-edge, s=9, c='green', marker='+', linewidths=3)
# plt.scatter(fovea_corner[1]-values+fs, fovea_corner[0]-edge+fs, s=9, c='green', marker='+', linewidths=3)
# plt.scatter(fovea_corner[1]-values, fovea_corner[0]-edge+fs, s=9, c='green', marker='+', linewidths=3)
# plt.scatter(fovea_corner[1]-values+fs, fovea_corner[0]-edge, s=9, c='green', marker='+', linewidths=3)
remove_axes()
plt.tight_layout(-1)
plt.show()
def rationale():
"""
Figure that illustrates rationale for fovea approach, i.e. diminishing returns
with increasing runtime via further iterations at a given downfactor, but
possibly too-high cost of using lower downfactor.
"""
# n_frames = 194
n_frames = 50
frame_down_factor = 1
source = KittiMultiViewSource(0)
full_shape = source.frame_ten[0].shape
frame_shape = downsample(source.frame_ten[0], frame_down_factor).shape
params = {
'data_weight': 0.16145115747533928,
'disc_max': 294.1504935618425,
'data_max': 32.024780646200725,
'laplacian_ksize': 3
}
# iterations = [1, 2, 3]
iterations = [1, 2, 3, 4, 5, 7, 10, 15]
mean_coarse_times = []
mean_coarse_costs = []
mean_fine_times = []
mean_fine_costs = []
for j in range(len(iterations)):
print(str(iterations[j]) + ' iterations')
coarse_times = []
coarse_costs = []
fine_times = []
fine_costs = []
for i in range(n_frames):
print(' frame ' + str(i))
source = KittiMultiViewSource(i)
true_points = source.get_ground_truth_points(occluded=False)
frame_down_factor = 1
coarse_time, coarse_cost = _evaluate_frame(source.frame_ten,
true_points,
frame_shape, full_shape,
frame_down_factor + 1,
frame_down_factor,
iterations[j], params)
fine_time, fine_cost = _evaluate_frame(source.frame_ten,
true_points, frame_shape,
full_shape,
frame_down_factor + 0,
frame_down_factor,
iterations[j], params)
coarse_times.append(coarse_time)
coarse_costs.append(coarse_cost)
fine_times.append(fine_time)
fine_costs.append(fine_cost)
mean_coarse_times.append(np.mean(coarse_times))
mean_coarse_costs.append(np.mean(coarse_costs))
mean_fine_times.append(np.mean(fine_times))
mean_fine_costs.append(np.mean(fine_costs))
print(mean_coarse_times)
print(mean_coarse_costs)
print(mean_fine_times)
print(mean_fine_costs)
plt.plot(mean_coarse_times,
mean_coarse_costs,
color='k',
marker='s',
markersize=12)
plt.plot(mean_fine_times,
mean_fine_costs,
color='k',
marker='o',
markersize=12)
plt.xlabel('Runtime (s)', fontsize=18)
plt.ylabel('Mean absolute disparity error (pixels)', fontsize=18)
plt.gca().tick_params(labelsize='18')
plt.show()
error_clip = 20
n_frames = 0
# for index in range(1):
# for index in [2]:
# for index in range(5,10):
# for index in range(17, 18):
# for index in range(27, 28):
# for index in range(30):
# for index in range(30, 60):
# for index in range(80):
for index in range(194):
if index in [31, 82, 114]: # missing frames
continue
source = KittiMultiViewSource(index, test=False, n_frames=n_frames)
full_shape = source.frame_ten[0].shape
frame_ten = [downsample(source.frame_ten[0], frame_down_factor),
downsample(source.frame_ten[1], frame_down_factor)]
frame_shape = frame_ten[0].shape
# fovea_shape = (frame_shape[0], frame_shape[1] - values)
try:
average_disp = source.get_average_disparity()
except IOError: # likely does not have a full 20 frames
print("Skipping index %d (lacks frames)" % index)
continue
average_disp = upsample_average_disp(
average_disp, frame_down_factor, frame_shape)
def test_foveal(frame_down_factor,
fovea_fraction,
fovea_n,
post_smooth=None,
**kwargs):
values = full_values / 2**frame_down_factor
mem_down_factor = 2 # relative to the frame down factor
shape = (len(fovea_n) + 1, n_test_frames)
times = np.zeros(shape)
unweighted_cost = np.zeros(shape)
weighted_cost = np.zeros(shape)
for i_frame, index in enumerate(range(n_test_frames)):
if index in [31, 82, 114]: # missing frames
continue
n_history_frames = 0
source = KittiMultiViewSource(index,
test=False,
n_frames=n_history_frames)
full_shape = source.frame_ten[0].shape
try:
frame_ten, frame_shape, average_disp, true_points = get_test_case(
source)
except IOError: # likely does not have a full 20 frames
print("Skipping index %d (lacks frames)" % index)
continue
frame_shape = frame_ten[0].shape
average_disp = upsample_average_disp(average_disp,
frame_down_factor,
frame_shape,
values=values)
true_disp = points_image(true_points,
frame_shape,
full_shape=full_shape)
true_disp_d = downsample(true_disp,
mem_down_factor)[:,
values / 2**mem_down_factor:]
average_disp_d = downsample(average_disp, mem_down_factor)
fovea_pixels = np.ceil(fovea_fraction * frame_shape[0] *
(frame_shape[1] - values))
if fovea_pixels > 0:
fovea_height = np.minimum(frame_shape[0],
np.ceil(fovea_pixels**.5))
fovea_width = np.minimum(frame_shape[1],
np.ceil(fovea_pixels / fovea_height))
fovea_shape = fovea_height, fovea_width
else:
fovea_shape = (0, 0)
# --- static fovea
params = {
'data_exp': 1.09821084614,
'data_max': 112.191597317,
'data_weight': 0.0139569211273,
'disc_max': 12.1301410452,
'laplacian_ksize': 3,
'smooth': 1.84510833504e-07
}
# params = {
# 'data_exp': 14.2348581842, 'data_max': 79101007093.4,
# 'data_weight': 0.000102496570364, 'disc_max': 4.93508276126,
# 'laplacian_ksize': 5, 'laplacian_scale': 0.38937704644,
# 'smooth': 0.00146126755993} # optimized for frame_down: 1, mem_down: 2, fovea_levels: 1
params.update(kwargs)
fovea_corner = ((frame_shape[0] - fovea_shape[0]) / 2,
(frame_shape[1] - fovea_shape[1]))
t = time.time()
foveal_disp = foveal_bp(frame_ten,
np.array(fovea_corner),
fovea_shape,
values=values,
post_smooth=post_smooth,
**params)
foveal_disp *= 2**frame_down_factor
times[0, i_frame] = time.time() - t
unweighted_cost[0, i_frame] = cost_on_points(foveal_disp[:, values:],
true_points,
full_shape=full_shape,
clip=error_clip)
weighted_cost[0, i_frame] = cost_on_points(foveal_disp[:, values:],
true_points,
average_disp,
full_shape=full_shape,
clip=error_clip)
# --- moving foveas
for i_fovea, fovea_n_i in enumerate(fovea_n):
foveal = Foveal(average_disp,
frame_down_factor,
mem_down_factor,
fovea_shape,
frame_shape,
values,
max_n_foveas=fovea_n_i,
**params)
foveal.post_smooth = post_smooth
if 0:
# use ground truth importance
pos_weights = get_position_weights(true_disp_d.shape)
importance = get_importance(pos_weights, average_disp_d,
true_disp_d)
importance[true_disp_d == 0] = 0
edge = 3
mask = np.zeros(importance.shape, dtype=bool)
mask[edge:-edge, edge:-edge] = 1
importance[~mask] = 0
# plt.figure()
# plt.imshow(true_disp_d)
# plt.colorbar()
# plt.figure()
# plt.imshow(importance)
# plt.colorbar()
# plt.show()
foveal_disp, fovea_corner = foveal.process_frame(
frame_ten, cost=importance)
else:
# use estimated importance
foveal_disp, fovea_corner = foveal.process_frame(frame_ten)
foveal_time = foveal.bp_time
unweighted_cost[i_fovea + 1,
i_frame] = cost_on_points(foveal_disp[:, values:],
true_points,
full_shape=full_shape,
clip=error_clip)
weighted_cost[i_fovea + 1,
i_frame] = cost_on_points(foveal_disp[:, values:],
true_points,
average_disp,
full_shape=full_shape,
clip=error_clip)
times[i_fovea + 1, i_frame] = foveal_time
return times, unweighted_cost, weighted_cost