def fovea_examples():
video = load_stereo_video(51, 4)
frame = video[3]
params = {'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425, 'data_max': 32.024780646200725, 'ksize': 3}
down_factor = 0
seed = np.zeros((0,0), dtype='uint8')
fig = plt.figure(1)
values = 128/2**down_factor
edge = 25
plt.subplot(211)
disp = foveal_bp(frame, (910, 200), seed, down_factor=down_factor, iters=5, **params)
plt.imshow(trim(disp, values, edge), vmin=0, vmax=values)
plt.scatter(910-values, 200-edge, s=100, c='white', marker='+', linewidths=2)
plt.gca().set_frame_on(False)
plt.gca().get_yaxis().set_visible(False)
plt.gca().get_xaxis().set_visible(False)
plt.subplot(212)
disp = foveal_bp(frame, (590, 50), seed, down_factor=down_factor, iters=5, **params)
plt.imshow(trim(disp, values, 25), vmin=0, vmax=values)
plt.scatter(590-values, 50-edge, s=100, c='white', marker='+', linewidths=2)
plt.gca().set_frame_on(False)
plt.gca().get_yaxis().set_visible(False)
plt.gca().get_xaxis().set_visible(False)
plt.tight_layout()
plt.show(block=True)
def fovea_examples():
video = load_stereo_video(51, 4)
frame = video[3]
params = {
'data_weight': 0.16145115747533928,
'disc_max': 294.1504935618425,
'data_max': 32.024780646200725,
'ksize': 3
}
down_factor = 0
seed = np.zeros((0, 0), dtype='uint8')
fig = plt.figure(1)
values = 128 / 2**down_factor
edge = 25
plt.subplot(211)
disp = foveal_bp(frame, (910, 200),
seed,
down_factor=down_factor,
iters=5,
**params)
plt.imshow(trim(disp, values, edge), vmin=0, vmax=values)
plt.scatter(910 - values,
200 - edge,
s=100,
c='white',
marker='+',
linewidths=2)
plt.gca().set_frame_on(False)
plt.gca().get_yaxis().set_visible(False)
plt.gca().get_xaxis().set_visible(False)
plt.subplot(212)
disp = foveal_bp(frame, (590, 50),
seed,
down_factor=down_factor,
iters=5,
**params)
plt.imshow(trim(disp, values, 25), vmin=0, vmax=values)
plt.scatter(590 - values,
50 - edge,
s=100,
c='white',
marker='+',
linewidths=2)
plt.gca().set_frame_on(False)
plt.gca().get_yaxis().set_visible(False)
plt.gca().get_xaxis().set_visible(False)
plt.tight_layout()
plt.show(block=True)
def process_frame(frame):
params = {
'data_exp': 1.09821084614,
'data_max': 82.191597317,
'data_weight': 0.0139569211273,
'disc_max': 15.1301410452
}
laplacian_ksize = 3
laplacian_scale = 0.5
iters = 5
left, right, top, bottom = values + 6, 6, 6, 6
frame = (pad(frame[0], left, right, top,
bottom), pad(frame[1], left, right, top, bottom))
down_factor = 1
values_coarse = values / 2**down_factor
img1d = downsample(frame[0], down_factor)
img2d = downsample(frame[1], down_factor)
if fast:
fovea_levels = 2
min_level = 1
else:
fovea_levels = 1
min_level = 0
fovea_corner = np.asarray((0, 0))
fovea_shape = np.asarray((0, 0))
disp = foveal_bp((img1d, img2d),
fovea_corner,
fovea_shape,
seed=None,
values=values_coarse,
iters=iters,
levels=5,
fovea_levels=fovea_levels,
min_level=min_level,
laplacian_ksize=1,
laplacian_scale=0.5,
post_smooth=None,
**params)
disp *= 2**down_factor
disp = upsample(disp, down_factor, frame[0].shape)
disp = unpad(disp, left, right, top, bottom)
# # this actually helps slightly but let's keep it simple and fast
# post_smooth = 3
# disp = cv2.GaussianBlur(disp.astype(np.float32), (9, 9), post_smooth)
# disp = np.round(disp).astype(np.uint8)
return disp
def seed_outside_fovea():
source = KittiSource(51, 5)
params = {'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425, 'data_max': 32.024780646200725, 'ksize': 3}
down_factor = 0
values = 128/2**down_factor
edge = 25
fn = 3 #frame number
seed = np.zeros((0,0), dtype='uint8')
disp_no_seed = foveal_bp(source.video[fn], 910, 200, seed, down_factor=down_factor, iters=5, **params)
plt.subplot(311)
plt.imshow(trim(disp_no_seed, values, edge), vmin=0, vmax=values)
plt.scatter(910-values, 200-edge, s=100, c='white', marker='+', linewidths=2)
remove_axes()
disp_previous_frame = foveal_bp(source.video[fn-1], 590, 50, seed, down_factor=down_factor, iters=5, **params)
mem = DisparityMemory(disp_previous_frame.shape, fovea_shape=(100,300), fill_method='smudge')
mem.remember(source.positions[fn-1], disp_previous_frame, fovea_centre=(50,590))
mem.remember_pos(source.positions[fn]) #TODO: change method name to move
seed = mem.transforms[0].astype('uint8')
disp_seed = foveal_bp(source.video[fn], 910, 200, seed, down_factor=down_factor, iters=5, **params)
plt.subplot(312)
plt.imshow(trim(seed, values, edge), vmin=0, vmax=values)
plt.scatter(910-values, 200-edge, s=100, c='white', marker='+', linewidths=0)
remove_axes()
plt.subplot(313)
plt.imshow(trim(disp_seed, values, edge), vmin=0, vmax=values)
plt.scatter(910-values, 200-edge, s=100, c='white', marker='+', linewidths=2)
remove_axes()
plt.tight_layout()
plt.show(block=True)
def process_frame(frame):
params = {
'data_exp': 1.09821084614, 'data_max': 82.191597317,
'data_weight': 0.0139569211273, 'disc_max': 15.1301410452}
laplacian_ksize = 3
laplacian_scale = 0.5
iters = 5
left, right, top, bottom = values+6, 6, 6, 6
frame = (pad(frame[0], left, right, top, bottom),
pad(frame[1], left, right, top, bottom))
down_factor = 1
values_coarse = values / 2**down_factor
img1d = downsample(frame[0], down_factor)
img2d = downsample(frame[1], down_factor)
if fast:
fovea_levels = 2
min_level = 1
else:
fovea_levels = 1
min_level = 0
fovea_corner = np.asarray((0,0))
fovea_shape = np.asarray((0,0))
disp = foveal_bp((img1d, img2d), fovea_corner, fovea_shape, seed=None,
values=values_coarse, iters=iters, levels=5, fovea_levels=fovea_levels, min_level=min_level,
laplacian_ksize=1, laplacian_scale=0.5, post_smooth=None, **params)
disp *= 2**down_factor
disp = upsample(disp, down_factor, frame[0].shape)
disp = unpad(disp, left, right, top, bottom)
# # this actually helps slightly but let's keep it simple and fast
# post_smooth = 3
# disp = cv2.GaussianBlur(disp.astype(np.float32), (9, 9), post_smooth)
# disp = np.round(disp).astype(np.uint8)
return disp
tic()
fine_disp = coarse_bp(frame, values=values, down_factor=0, iters=iters, **params)
fine_disp *= 2**frame_down_factor
toc()
# fovea_corners = (60, 360)
# fovea_shapes = (80, 80)
fovea_corners = [(60, 160), (60, 360)]
fovea_shapes = [(80, 80), (80, 80)]
fovea_corners = np.array(fovea_corners, ndmin=2)
fovea_shapes = np.array(fovea_shapes, ndmin=2)
tic()
foveal_disp = foveal_bp(frame, fovea_corners, fovea_shapes, values=values, iters=iters, **params)
foveal_disp *= 2**frame_down_factor
toc()
coarse_error = error_on_points(points, coarse_disp, kind='abs')
fine_error = error_on_points(points, fine_disp, kind='abs')
foveal_error = error_on_points(points, foveal_disp, kind='abs')
print("Errors (coarse, fine, foveal): %f, %f, %f" % (
coarse_error, fine_error, foveal_error))
# plt.ion()
plt.figure(1)
plt.clf()
axes = [plt.subplot(4, 2, i+1) for i in range(7)]
axes[0].imshow(frame[0], cmap='gray', interpolation='none')
**params)
fine_disp *= 2**frame_down_factor
toc()
# fovea_corners = (60, 360)
# fovea_shapes = (80, 80)
fovea_corners = [(60, 160), (60, 360)]
fovea_shapes = [(80, 80), (80, 80)]
fovea_corners = np.array(fovea_corners, ndmin=2)
fovea_shapes = np.array(fovea_shapes, ndmin=2)
tic()
foveal_disp = foveal_bp(frame,
fovea_corners,
fovea_shapes,
values=values,
iters=iters,
**params)
foveal_disp *= 2**frame_down_factor
toc()
coarse_error = error_on_points(points, coarse_disp, kind='abs')
fine_error = error_on_points(points, fine_disp, kind='abs')
foveal_error = error_on_points(points, foveal_disp, kind='abs')
print("Errors (coarse, fine, foveal): %f, %f, %f" %
(coarse_error, fine_error, foveal_error))
# plt.ion()
plt.figure(1)
plt.clf()
axes = [plt.subplot(4, 2, i + 1) for i in range(7)]
def process_frame(self, pos, frame):
start_time = time.time()
self.disparity_memory.move(pos)
if self.n_past_fovea > 0:
self.fovea_memory.move(pos)
if self.use_uncertainty:
self.uncertainty_memory.move(pos)
if self.verbose:
print('move time: ' + str(time.time() - start_time))
# 1. Decide where to put fovea and move it there:
if self.disparity_memory.n > 0 and len(
self.disparity_memory.transforms) == 0:
fovea_corner = (np.array(self.frame_shape) -
np.array(self.fovea_shape)) / 2
assert all(fovea_corner >= 0)
else:
# a) Transform disparity from previous frame and calculate importance
if self.disparity_memory.n > 0:
prior_disparity = self.disparity_memory.transforms[
0] #in current frame coords
else:
# TODO: we don't have GPS for multiview, so we're temporarily replacing past estimate with coarse
# estimate from current frame
params = {
'data_weight': 0.16145115747533928,
'disc_max': 294.1504935618425,
'data_max': 32.024780646200725,
'laplacian_ksize': 3
}
prior_disparity = coarse_bp(frame,
down_factor=self.mem_down_factor,
iters=5,
values=self.values,
**params)
prior_disparity *= self.frame_step
prior_disparity = prior_disparity[:,
self.values / self.mem_step:]
importance = self._uc.get_importance(prior_disparity)
# b) Transform uncertainty from previous frame and multiply by importance
if self.use_uncertainty:
uncertainty = np.ones_like(importance)
if len(self.uncertainty_memory.transforms) > 0:
uncertainty = self.uncertainty_memory.transforms[0]
cost = importance * uncertainty
else:
cost = importance
assert cost.shape == self.memory_shape
# c) Find region of highest cost and put fovea there
# mem_fovea_shape = np.array(self.fovea_shape) / self.mem_step
# fovea_corner = _choose_fovea(cost, mem_fovea_shape, 0)
# fovea_corner = np.array(fovea_corner) * self.mem_step + np.array([0, self.values])
mem_fovea_shape = np.array(self.fovea_shape) / self.mem_step
fovea_corners, mem_fovea_shape = _choose_foveas(
cost, mem_fovea_shape, self.values / self.mem_step,
self.max_n_foveas)
# ### debug plot
# print(fovea_corners)
# print(mem_fovea_shape)
# plt.imshow(cost, vmin=0, vmax=128/self.mem_step)
# for (fi, fj) in fovea_corners:
# fm, fn = mem_fovea_shape
# plt.plot([fj, fj+fn, fj+fn, fj, fj], [fi, fi, fi+fm, fi+fm, fi], 'white')
# plt.colorbar()
# plt.show()
# ###
# rescale shape and corners and trim fovea to image ...
fovea_shape = np.array(
mem_fovea_shape
) * self.mem_step # this isn't redundant because _choose_foveas chooses the multifovea shape
fovea_corners = np.array(fovea_corners, dtype='int32')
for i in range(len(fovea_corners)):
fovea_corners[i] = np.array(
fovea_corners[i]) * self.mem_step + np.array(
[0, self.values])
# fovea corner can be slightly too large, due to rounding errors
fovea_max = np.array(self.frame_shape) - fovea_shape
fovea_corners[i] = np.minimum(fovea_corners[i], fovea_max)
assert fovea_corners[i][0] >= 0 and fovea_corners[i][
1] >= self.values
assert all(fovea_corners[i] + fovea_shape <= self.frame_shape)
if self.verbose:
print('choose time: ' + str(time.time() - start_time))
# 2. Calculate disparity and store in memory:
if len(self.fovea_memory.transforms) == 0:
seed = np.zeros((0, 0), dtype='uint8')
else:
seed = np.zeros(self.frame_shape, dtype='uint8')
for t in self.fovea_memory.transforms:
seed += t
if self.verbose:
print('seed time: ' + str(time.time() - start_time))
# --- fovea boundaries in frame coordinates ...
bp_time = time.time()
disp = foveal_bp(frame,
fovea_corners,
fovea_shape,
seed,
values=self.values,
**self.params)
self.bp_time = time.time() - bp_time
# disp = coarse_bp(frame, down_factor=1, iters=3, values=self.values, **self.params)
# disp = cv2.pyrUp(disp)[:self.frame_shape[0], :self.frame_shape[1]]
# keep all disparities in full image coordinates
disp *= self.frame_step
if self.verbose:
print('BP time: ' + str(time.time() - start_time))
# --- downsample and remember disparity
downsampled = downsample(disp[:, self.values:], self.mem_down_factor)
assert downsampled.shape == self.memory_shape
self.disparity_memory.remember(pos, downsampled)
if self.n_past_fovea > 0:
self.fovea_memory.remember(pos, disp, fovea_corner=fovea_corner)
# 3. Calculate uncertainty and store in memory
if self.use_uncertainty and len(self.disparity_memory.transforms) > 0:
prior_disparity = self.disparity_memory.transforms[0]
uncertainty = np.abs(downsampled - prior_disparity)
self.uncertainty_memory.remember(pos, uncertainty)
if self.verbose:
print('finish time: ' + str(time.time() - start_time))
#TODO: use multiple fovea_corners in history
return disp, fovea_corners
def process_frame(self, frame, cost=None):
start_time = time.time()
if cost is None:
# Use coarse BP run to get importance for current frame
# params = {
# 'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425,
# 'data_max': 32.024780646200725, 'laplacian_ksize': 3}
prior_params = {
'data_exp': 1.09821084614,
'data_max': 112.191597317,
'data_weight': 0.0139569211273,
'disc_max': 12.1301410452,
'laplacian_ksize': 3,
'smooth': 1.84510833504e-07
}
prior_disparity = coarse_bp(frame,
down_factor=self.mem_down_factor,
iters=5,
values=self.values,
**prior_params)
prior_disparity *= self.frame_step
prior_disparity = prior_disparity[:, self.values / self.mem_step:]
cost = self._uc.get_importance(prior_disparity)
assert cost.shape == self.average_disparity.shape
# c) Find region of highest cost and put fovea there
# mem_fovea_shape = np.array(self.fovea_shape) / self.mem_step
# fovea_corner = _choose_fovea(cost, mem_fovea_shape, 0)
# fovea_corner = np.array(fovea_corner) * self.mem_step + np.array([0, self.values])
mem_fovea_shape = np.array(self.fovea_shape) / self.mem_step
fovea_corners, mem_fovea_shape = _choose_foveas(
cost, mem_fovea_shape, self.values / self.mem_step,
self.max_n_foveas)
# ### debug plot
# print(fovea_corners)
# print(mem_fovea_shape)
# plt.imshow(cost, vmin=0, vmax=128/self.mem_step)
# for (fi, fj) in fovea_corners:
# fm, fn = mem_fovea_shape
# plt.plot([fj, fj+fn, fj+fn, fj, fj], [fi, fi, fi+fm, fi+fm, fi], 'white')
# plt.colorbar()
# plt.show()
# ###
# rescale shape and corners and trim fovea to image ...
fovea_shape = np.array(
mem_fovea_shape
) * self.mem_step # this isn't redundant because _choose_foveas chooses the multifovea shape
fovea_corners = np.array(fovea_corners, dtype='int32')
for i in range(len(fovea_corners)):
fovea_corners[i] = np.array(
fovea_corners[i]) * self.mem_step + np.array([0, self.values])
# fovea corner can be slightly too large, due to rounding errors
fovea_max = np.array(self.frame_shape) - fovea_shape
fovea_corners[i] = np.minimum(fovea_corners[i], fovea_max)
assert fovea_corners[i][0] >= 0 and fovea_corners[i][
1] >= self.values
assert all(fovea_corners[i] + fovea_shape <= self.frame_shape)
# --- fovea boundaries in frame coordinates ...
bp_time = time.time()
disp = foveal_bp(frame,
fovea_corners,
fovea_shape,
values=self.values,
post_smooth=self.post_smooth,
**self.params)
disp *= self.frame_step
self.bp_time = time.time() - bp_time
return disp, fovea_corners
def process_frame(self, pos, frame):
start_time = time.time()
self.disparity_memory.move(pos)
if self.n_past_fovea > 0:
self.fovea_memory.move(pos)
if self.use_uncertainty:
self.uncertainty_memory.move(pos)
if self.verbose:
print('move time: ' + str(time.time() - start_time))
# 1. Decide where to put fovea and move it there:
if self.disparity_memory.n > 0 and len(self.disparity_memory.transforms) == 0:
fovea_corner = (
np.array(self.frame_shape) - np.array(self.fovea_shape)) / 2
assert all(fovea_corner >= 0)
else:
# a) Transform disparity from previous frame and calculate importance
if self.disparity_memory.n > 0:
prior_disparity = self.disparity_memory.transforms[0] #in current frame coords
else:
# TODO: we don't have GPS for multiview, so we're temporarily replacing past estimate with coarse
# estimate from current frame
params = {
'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425,
'data_max': 32.024780646200725, 'laplacian_ksize': 3}
prior_disparity = coarse_bp(frame, down_factor=self.mem_down_factor, iters=5, values=self.values, **params)
prior_disparity *= self.frame_step
prior_disparity = prior_disparity[:,self.values/self.mem_step:]
importance = self._uc.get_importance(prior_disparity)
# b) Transform uncertainty from previous frame and multiply by importance
if self.use_uncertainty:
uncertainty = np.ones_like(importance)
if len(self.uncertainty_memory.transforms) > 0:
uncertainty = self.uncertainty_memory.transforms[0]
cost = importance * uncertainty
else:
cost = importance
assert cost.shape == self.memory_shape
# c) Find region of highest cost and put fovea there
# mem_fovea_shape = np.array(self.fovea_shape) / self.mem_step
# fovea_corner = _choose_fovea(cost, mem_fovea_shape, 0)
# fovea_corner = np.array(fovea_corner) * self.mem_step + np.array([0, self.values])
mem_fovea_shape = np.array(self.fovea_shape) / self.mem_step
fovea_corners, mem_fovea_shape = _choose_foveas(
cost, mem_fovea_shape, self.values/self.mem_step, self.max_n_foveas)
# ### debug plot
# print(fovea_corners)
# print(mem_fovea_shape)
# plt.imshow(cost, vmin=0, vmax=128/self.mem_step)
# for (fi, fj) in fovea_corners:
# fm, fn = mem_fovea_shape
# plt.plot([fj, fj+fn, fj+fn, fj, fj], [fi, fi, fi+fm, fi+fm, fi], 'white')
# plt.colorbar()
# plt.show()
# ###
# rescale shape and corners and trim fovea to image ...
fovea_shape = np.array(mem_fovea_shape) * self.mem_step # this isn't redundant because _choose_foveas chooses the multifovea shape
fovea_corners = np.array(fovea_corners, dtype='int32')
for i in range(len(fovea_corners)):
fovea_corners[i] = np.array(fovea_corners[i]) * self.mem_step + np.array([0, self.values])
# fovea corner can be slightly too large, due to rounding errors
fovea_max = np.array(self.frame_shape) - fovea_shape
fovea_corners[i] = np.minimum(fovea_corners[i], fovea_max)
assert fovea_corners[i][0] >= 0 and fovea_corners[i][1] >= self.values
assert all(fovea_corners[i] + fovea_shape <= self.frame_shape)
if self.verbose:
print('choose time: ' + str(time.time() - start_time))
# 2. Calculate disparity and store in memory:
if len(self.fovea_memory.transforms) == 0:
seed = np.zeros((0,0), dtype='uint8')
else:
seed = np.zeros(self.frame_shape, dtype='uint8')
for t in self.fovea_memory.transforms:
seed += t
if self.verbose:
print('seed time: ' + str(time.time() - start_time))
# --- fovea boundaries in frame coordinates ...
bp_time = time.time()
disp = foveal_bp(
frame, fovea_corners, fovea_shape, seed,
values=self.values, **self.params)
self.bp_time = time.time() - bp_time
# disp = coarse_bp(frame, down_factor=1, iters=3, values=self.values, **self.params)
# disp = cv2.pyrUp(disp)[:self.frame_shape[0], :self.frame_shape[1]]
# keep all disparities in full image coordinates
disp *= self.frame_step
if self.verbose:
print('BP time: ' + str(time.time() - start_time))
# --- downsample and remember disparity
downsampled = downsample(disp[:,self.values:], self.mem_down_factor)
assert downsampled.shape == self.memory_shape
self.disparity_memory.remember(pos, downsampled)
if self.n_past_fovea > 0:
self.fovea_memory.remember(pos, disp, fovea_corner=fovea_corner)
# 3. Calculate uncertainty and store in memory
if self.use_uncertainty and len(self.disparity_memory.transforms) > 0:
prior_disparity = self.disparity_memory.transforms[0]
uncertainty = np.abs(downsampled - prior_disparity)
self.uncertainty_memory.remember(pos, uncertainty)
if self.verbose:
print('finish time: ' + str(time.time() - start_time))
#TODO: use multiple fovea_corners in history
return disp, fovea_corners
def seed_outside_fovea():
source = KittiSource(51, 5)
params = {
'data_weight': 0.16145115747533928,
'disc_max': 294.1504935618425,
'data_max': 32.024780646200725,
'ksize': 3
}
down_factor = 0
values = 128 / 2**down_factor
edge = 25
fn = 3 #frame number
seed = np.zeros((0, 0), dtype='uint8')
disp_no_seed = foveal_bp(source.video[fn],
910,
200,
seed,
down_factor=down_factor,
iters=5,
**params)
plt.subplot(311)
plt.imshow(trim(disp_no_seed, values, edge), vmin=0, vmax=values)
plt.scatter(910 - values,
200 - edge,
s=100,
c='white',
marker='+',
linewidths=2)
remove_axes()
disp_previous_frame = foveal_bp(source.video[fn - 1],
590,
50,
seed,
down_factor=down_factor,
iters=5,
**params)
mem = DisparityMemory(disp_previous_frame.shape,
fovea_shape=(100, 300),
fill_method='smudge')
mem.remember(source.positions[fn - 1],
disp_previous_frame,
fovea_centre=(50, 590))
mem.remember_pos(source.positions[fn]) #TODO: change method name to move
seed = mem.transforms[0].astype('uint8')
disp_seed = foveal_bp(source.video[fn],
910,
200,
seed,
down_factor=down_factor,
iters=5,
**params)
plt.subplot(312)
plt.imshow(trim(seed, values, edge), vmin=0, vmax=values)
plt.scatter(910 - values,
200 - edge,
s=100,
c='white',
marker='+',
linewidths=0)
remove_axes()
plt.subplot(313)
plt.imshow(trim(disp_seed, values, edge), vmin=0, vmax=values)
plt.scatter(910 - values,
200 - edge,
s=100,
c='white',
marker='+',
linewidths=2)
remove_axes()
plt.tight_layout()
plt.show(block=True)
def test_foveal(frame_down_factor,
fovea_fraction,
fovea_n,
post_smooth=None,
**kwargs):
values = full_values / 2**frame_down_factor
mem_down_factor = 2 # relative to the frame down factor
shape = (len(fovea_n) + 1, n_test_frames)
times = np.zeros(shape)
unweighted_cost = np.zeros(shape)
weighted_cost = np.zeros(shape)
for i_frame, index in enumerate(range(n_test_frames)):
if index in [31, 82, 114]: # missing frames
continue
n_history_frames = 0
source = KittiMultiViewSource(index,
test=False,
n_frames=n_history_frames)
full_shape = source.frame_ten[0].shape
try:
frame_ten, frame_shape, average_disp, true_points = get_test_case(
source)
except IOError: # likely does not have a full 20 frames
print("Skipping index %d (lacks frames)" % index)
continue
frame_shape = frame_ten[0].shape
average_disp = upsample_average_disp(average_disp,
frame_down_factor,
frame_shape,
values=values)
true_disp = points_image(true_points,
frame_shape,
full_shape=full_shape)
true_disp_d = downsample(true_disp,
mem_down_factor)[:,
values / 2**mem_down_factor:]
average_disp_d = downsample(average_disp, mem_down_factor)
fovea_pixels = np.ceil(fovea_fraction * frame_shape[0] *
(frame_shape[1] - values))
if fovea_pixels > 0:
fovea_height = np.minimum(frame_shape[0],
np.ceil(fovea_pixels**.5))
fovea_width = np.minimum(frame_shape[1],
np.ceil(fovea_pixels / fovea_height))
fovea_shape = fovea_height, fovea_width
else:
fovea_shape = (0, 0)
# --- static fovea
params = {
'data_exp': 1.09821084614,
'data_max': 112.191597317,
'data_weight': 0.0139569211273,
'disc_max': 12.1301410452,
'laplacian_ksize': 3,
'smooth': 1.84510833504e-07
}
# params = {
# 'data_exp': 14.2348581842, 'data_max': 79101007093.4,
# 'data_weight': 0.000102496570364, 'disc_max': 4.93508276126,
# 'laplacian_ksize': 5, 'laplacian_scale': 0.38937704644,
# 'smooth': 0.00146126755993} # optimized for frame_down: 1, mem_down: 2, fovea_levels: 1
params.update(kwargs)
fovea_corner = ((frame_shape[0] - fovea_shape[0]) / 2,
(frame_shape[1] - fovea_shape[1]))
t = time.time()
foveal_disp = foveal_bp(frame_ten,
np.array(fovea_corner),
fovea_shape,
values=values,
post_smooth=post_smooth,
**params)
foveal_disp *= 2**frame_down_factor
times[0, i_frame] = time.time() - t
unweighted_cost[0, i_frame] = cost_on_points(foveal_disp[:, values:],
true_points,
full_shape=full_shape,
clip=error_clip)
weighted_cost[0, i_frame] = cost_on_points(foveal_disp[:, values:],
true_points,
average_disp,
full_shape=full_shape,
clip=error_clip)
# --- moving foveas
for i_fovea, fovea_n_i in enumerate(fovea_n):
foveal = Foveal(average_disp,
frame_down_factor,
mem_down_factor,
fovea_shape,
frame_shape,
values,
max_n_foveas=fovea_n_i,
**params)
foveal.post_smooth = post_smooth
if 0:
# use ground truth importance
pos_weights = get_position_weights(true_disp_d.shape)
importance = get_importance(pos_weights, average_disp_d,
true_disp_d)
importance[true_disp_d == 0] = 0
edge = 3
mask = np.zeros(importance.shape, dtype=bool)
mask[edge:-edge, edge:-edge] = 1
importance[~mask] = 0
# plt.figure()
# plt.imshow(true_disp_d)
# plt.colorbar()
# plt.figure()
# plt.imshow(importance)
# plt.colorbar()
# plt.show()
foveal_disp, fovea_corner = foveal.process_frame(
frame_ten, cost=importance)
else:
# use estimated importance
foveal_disp, fovea_corner = foveal.process_frame(frame_ten)
foveal_time = foveal.bp_time
unweighted_cost[i_fovea + 1,
i_frame] = cost_on_points(foveal_disp[:, values:],
true_points,
full_shape=full_shape,
clip=error_clip)
weighted_cost[i_fovea + 1,
i_frame] = cost_on_points(foveal_disp[:, values:],
true_points,
average_disp,
full_shape=full_shape,
clip=error_clip)
times[i_fovea + 1, i_frame] = foveal_time
return times, unweighted_cost, weighted_cost