def objective(args):
error = 0.0
for source, frame_ten, true_points in sources:
full_shape = source.frame_ten[0].shape
frame_shape = frame_ten[0].shape
frame_ten = [downsample(source.frame_ten[0], frame_down_factor),
downsample(source.frame_ten[1], frame_down_factor)]
# --- BP
disp = coarse_bp(
frame_ten, values=values, down_factor=0, iters=iters, **args)
disp *= 2**frame_down_factor
# --- compute error
cost_i = cost_on_points(disp[:,values:], true_points, full_shape=full_shape)
error += cost_i
if 0:
print(cost_i, error)
plt.figure()
rows, cols = 2, 1
plt.subplot(rows, cols, 1)
plt.imshow(frame_ten[0], cmap='gray')
plt.subplot(rows, cols, 2)
plt.imshow(disp, vmin=0, vmax=64)
# plt.imshow(disp, vmin=0, vmax=full_values)
plt.show()
error /= len(sources)
errors[arg_key(args)] = error
args_s = "{%s}" % ', '.join("%r: %s" % (k, args[k]) for k in sorted(space))
print("%s: %s" % (error, args_s))
return error
def eval_coarse(frame_ten, frame_shape, values=values):
args = {
'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425,
'data_max': 32.024780646200725, 'laplacian_ksize': 3}
coarse_time = time.time()
coarse_disp = coarse_bp(frame_ten, values=values, down_factor=1,
iters=iters, **args)
coarse_disp = cv2.pyrUp(coarse_disp)[:frame_shape[0],:frame_shape[1]]
coarse_disp *= 2**frame_down_factor
coarse_time = time.time() - coarse_time
return coarse_disp, coarse_time
def estimate_coarse_error(down_factor, iters, video, n_disp, drive, n_frames=20, **params):
# TODO: use fine BP as ground truth?
coarse_error = []
for i in range(n_frames):
print(i)
frame = video[i]
tic('coarse')
coarse_disp = coarse_bp(frame, down_factor=down_factor, iters=iters, **params)
toc()
xyd = load_disparity_points(drive, i)
coarse_error.append(error_on_points(xyd, coarse_disp, n_disp, kind='close'))
return np.mean(coarse_error)
def _evaluate_frame(frame, true_points, frame_shape, full_shape, down_factor, frame_down_factor, iters, params):
values = 128/2**frame_down_factor
start_time = time.time()
coarse_disp = coarse_bp(frame, down_factor=down_factor,
iters=iters, values=128, **params)
disp = expand_coarse(coarse_disp, down_factor - frame_down_factor)
disp = disp[:frame_shape[0],:frame_shape[1]]
elapsed_time = time.time() - start_time
# unweighted_cost.append(cost_on_points(coarse_disp[:, values:], true_points, full_shape=full_shape))
# true_points = source.true_points[frame_num]
return elapsed_time, cost_on_points(disp[:,values:], true_points, full_shape=full_shape)
def calc_ground_truth(frame, n_disp, down_factor=0, iters=50):
params = {
'data_weight': 0.16145115747533928,
'disc_max': 294.1504935618425,
'data_max': 32.024780646200725,
'laplacian_ksize': 1
}
gt = coarse_bp(frame,
down_factor=down_factor,
iters=iters,
values=n_disp,
**params)
n_coarse_disp = n_disp / 2**down_factor
gt = gt[:, n_coarse_disp:]
return gt
def eval_fine(frame_ten, values=values):
# args = {
# 'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425,
# 'data_max': 32.024780646200725, 'laplacian_ksize': 3}
args = {'data_exp': 1.09821084614, 'data_max': 112.191597317,
'data_weight': 0.0139569211273, 'disc_max': 12.1301410452,
'laplacian_ksize': 3, 'smooth': 1.84510833504e-07}
# args = {
# 'data_exp': 0.697331027724, 'data_max': 49046.1429149,
# 'data_weight': 0.015291472002, 'disc_max': 599730.981833,
# 'laplacian_ksize': 5, 'laplacian_scale': 0.103881528095,
# 'post_smooth': 2.31047577912, 'smooth': 1.30343003387e-05}
fine_time = time.time()
fine_disp = coarse_bp(frame_ten, values=values, down_factor=0,
iters=iters, **args)
fine_disp *= 2**frame_down_factor
fine_time = time.time() - fine_time
return fine_disp, fine_time
def _evaluate_frame(frame, true_points, frame_shape, full_shape, down_factor,
frame_down_factor, iters, params):
values = 128 / 2**frame_down_factor
start_time = time.time()
coarse_disp = coarse_bp(frame,
down_factor=down_factor,
iters=iters,
values=128,
**params)
disp = expand_coarse(coarse_disp, down_factor - frame_down_factor)
disp = disp[:frame_shape[0], :frame_shape[1]]
elapsed_time = time.time() - start_time
# unweighted_cost.append(cost_on_points(coarse_disp[:, values:], true_points, full_shape=full_shape))
# true_points = source.true_points[frame_num]
return elapsed_time, cost_on_points(disp[:, values:],
true_points,
full_shape=full_shape)
def objective(args):
error = 0.0
for source, frame_ten, true_points in sources:
full_shape = source.frame_ten[0].shape
frame_shape = frame_ten[0].shape
frame_ten = [
downsample(source.frame_ten[0], frame_down_factor),
downsample(source.frame_ten[1], frame_down_factor)
]
# --- BP
disp = coarse_bp(frame_ten,
values=values,
down_factor=0,
iters=iters,
**args)
disp *= 2**frame_down_factor
# --- compute error
cost_i = cost_on_points(disp[:, values:],
true_points,
full_shape=full_shape)
error += cost_i
if 0:
print(cost_i, error)
plt.figure()
rows, cols = 2, 1
plt.subplot(rows, cols, 1)
plt.imshow(frame_ten[0], cmap='gray')
plt.subplot(rows, cols, 2)
plt.imshow(disp, vmin=0, vmax=64)
# plt.imshow(disp, vmin=0, vmax=full_values)
plt.show()
error /= len(sources)
errors[arg_key(args)] = error
args_s = "{%s}" % ', '.join("%r: %s" % (k, args[k]) for k in sorted(space))
print("%s: %s" % (error, args_s))
return error
iters = 3
params = {
'data_weight': 0.16145115747533928,
'disc_max': 294.1504935618425,
'data_max': 32.024780646200725,
'ksize': 3
}
frame = (downsample(frame[0],
frame_down_factor), downsample(frame[1],
frame_down_factor))
tic()
coarse_disp = coarse_bp(frame,
values=values,
down_factor=1,
iters=iters,
**params)
coarse_disp *= 2**frame_down_factor
toc()
tic()
fine_disp = coarse_bp(frame,
values=values,
down_factor=0,
iters=iters,
**params)
fine_disp *= 2**frame_down_factor
toc()
# fovea_corners = (60, 360)
def process_frame(self, pos, frame):
start_time = time.time()
self.disparity_memory.move(pos)
if self.n_past_fovea > 0:
self.fovea_memory.move(pos)
if self.use_uncertainty:
self.uncertainty_memory.move(pos)
if self.verbose:
print('move time: ' + str(time.time() - start_time))
# 1. Decide where to put fovea and move it there:
if self.disparity_memory.n > 0 and len(
self.disparity_memory.transforms) == 0:
fovea_corner = (np.array(self.frame_shape) -
np.array(self.fovea_shape)) / 2
assert all(fovea_corner >= 0)
else:
# a) Transform disparity from previous frame and calculate importance
if self.disparity_memory.n > 0:
prior_disparity = self.disparity_memory.transforms[
0] #in current frame coords
else:
# TODO: we don't have GPS for multiview, so we're temporarily replacing past estimate with coarse
# estimate from current frame
params = {
'data_weight': 0.16145115747533928,
'disc_max': 294.1504935618425,
'data_max': 32.024780646200725,
'laplacian_ksize': 3
}
prior_disparity = coarse_bp(frame,
down_factor=self.mem_down_factor,
iters=5,
values=self.values,
**params)
prior_disparity *= self.frame_step
prior_disparity = prior_disparity[:,
self.values / self.mem_step:]
importance = self._uc.get_importance(prior_disparity)
# b) Transform uncertainty from previous frame and multiply by importance
if self.use_uncertainty:
uncertainty = np.ones_like(importance)
if len(self.uncertainty_memory.transforms) > 0:
uncertainty = self.uncertainty_memory.transforms[0]
cost = importance * uncertainty
else:
cost = importance
assert cost.shape == self.memory_shape
# c) Find region of highest cost and put fovea there
# mem_fovea_shape = np.array(self.fovea_shape) / self.mem_step
# fovea_corner = _choose_fovea(cost, mem_fovea_shape, 0)
# fovea_corner = np.array(fovea_corner) * self.mem_step + np.array([0, self.values])
mem_fovea_shape = np.array(self.fovea_shape) / self.mem_step
fovea_corners, mem_fovea_shape = _choose_foveas(
cost, mem_fovea_shape, self.values / self.mem_step,
self.max_n_foveas)
# ### debug plot
# print(fovea_corners)
# print(mem_fovea_shape)
# plt.imshow(cost, vmin=0, vmax=128/self.mem_step)
# for (fi, fj) in fovea_corners:
# fm, fn = mem_fovea_shape
# plt.plot([fj, fj+fn, fj+fn, fj, fj], [fi, fi, fi+fm, fi+fm, fi], 'white')
# plt.colorbar()
# plt.show()
# ###
# rescale shape and corners and trim fovea to image ...
fovea_shape = np.array(
mem_fovea_shape
) * self.mem_step # this isn't redundant because _choose_foveas chooses the multifovea shape
fovea_corners = np.array(fovea_corners, dtype='int32')
for i in range(len(fovea_corners)):
fovea_corners[i] = np.array(
fovea_corners[i]) * self.mem_step + np.array(
[0, self.values])
# fovea corner can be slightly too large, due to rounding errors
fovea_max = np.array(self.frame_shape) - fovea_shape
fovea_corners[i] = np.minimum(fovea_corners[i], fovea_max)
assert fovea_corners[i][0] >= 0 and fovea_corners[i][
1] >= self.values
assert all(fovea_corners[i] + fovea_shape <= self.frame_shape)
if self.verbose:
print('choose time: ' + str(time.time() - start_time))
# 2. Calculate disparity and store in memory:
if len(self.fovea_memory.transforms) == 0:
seed = np.zeros((0, 0), dtype='uint8')
else:
seed = np.zeros(self.frame_shape, dtype='uint8')
for t in self.fovea_memory.transforms:
seed += t
if self.verbose:
print('seed time: ' + str(time.time() - start_time))
# --- fovea boundaries in frame coordinates ...
bp_time = time.time()
disp = foveal_bp(frame,
fovea_corners,
fovea_shape,
seed,
values=self.values,
**self.params)
self.bp_time = time.time() - bp_time
# disp = coarse_bp(frame, down_factor=1, iters=3, values=self.values, **self.params)
# disp = cv2.pyrUp(disp)[:self.frame_shape[0], :self.frame_shape[1]]
# keep all disparities in full image coordinates
disp *= self.frame_step
if self.verbose:
print('BP time: ' + str(time.time() - start_time))
# --- downsample and remember disparity
downsampled = downsample(disp[:, self.values:], self.mem_down_factor)
assert downsampled.shape == self.memory_shape
self.disparity_memory.remember(pos, downsampled)
if self.n_past_fovea > 0:
self.fovea_memory.remember(pos, disp, fovea_corner=fovea_corner)
# 3. Calculate uncertainty and store in memory
if self.use_uncertainty and len(self.disparity_memory.transforms) > 0:
prior_disparity = self.disparity_memory.transforms[0]
uncertainty = np.abs(downsampled - prior_disparity)
self.uncertainty_memory.remember(pos, uncertainty)
if self.verbose:
print('finish time: ' + str(time.time() - start_time))
#TODO: use multiple fovea_corners in history
return disp, fovea_corners
]
true_disp = downsample(source.ground_truth[i], frame_down_factor)
true_points = source.true_points[i]
# --- coarse
if run_coarse:
coarse_params = {
'data_weight': 0.16145115747533928,
'disc_max': 294.1504935618425,
'data_max': 32.024780646200725,
'laplacian_ksize': 3
}
coarse_time = time.time()
coarse_disp = coarse_bp(frame,
down_factor=1,
iters=3,
values=values,
**coarse_params)
coarse_disp = cv2.pyrUp(
coarse_disp)[:frame_shape[0], :frame_shape[1]]
coarse_disp *= 2
coarse_time = time.time() - coarse_time
append_table('coarse', coarse_disp[:, values:], true_disp,
true_points)
times['coarse'].append(coarse_time)
# --- fine
if run_fine:
fine_params = {
'data_weight': 0.16145115747533928,
iframe = 50
frame = load_stereo_frame(idrive, iframe)
points = load_disparity_points(idrive, iframe)
n_disp = 128
down_factor = 2
params = {
'data_weight': 0.16145115747533928,
'disc_max': 294.1504935618425,
'data_max': 32.024780646200725,
'ksize': 3
}
tic()
disp = coarse_bp(frame, values=n_disp, down_factor=down_factor, **params)
toc()
error = error_on_points(points, disp, kind='abs')
print("Error: %f" % error)
# plt.ion()
plt.figure(1)
plt.clf()
axes = [plt.subplot(3, 1, i + 1) for i in range(3)]
frame = (downsample(frame[0], down_factor), downsample(frame[1], down_factor))
axes[0].imshow(frame[0], cmap='gray', interpolation='none')
img0 = laplacian(frame[0], ksize=params['ksize'])
axes[1].imshow(img0, cmap='gray', interpolation='none')
axes[2].imshow(disp, vmin=0, vmax=n_disp, interpolation='none')
plt.show()
def time_coarse(down_factor, video, iters):
before = time.time()
n_frames = 3
for i in range(n_frames):
coarse_bp(video[i], down_factor=down_factor, iters=iters)
return (time.time() - before) / n_frames
def estimate_fovea_error(coarse_down_factor, coarse_iters, fovea_down_factor, fovea_iters, fovea_shape, video, positions, drive, n_frames=20):
fovea_ij = 200, 600 #note: ground truth only available in lower part of frame
#fovea_ij = 40, 100
low0 = coarse_bp(video[0], down_factor=coarse_down_factor, iters=coarse_iters)
high0 = coarse_bp(video[0], down_factor=fovea_down_factor, iters=fovea_iters)
high1 = fovea_bp(video[0], fovea_ij, fovea_shape, low0, down_factor=fovea_down_factor, iters=fovea_iters)
# high1 = np.zeros(full_shape, dtype=int)
# high1 = fine_bp(initial)
coarse_shape = np.asarray(low0.shape)
fine_shape = np.asarray(high0.shape)
# fine_shape = full_shape
fine_full_ratio = fine_shape / np.asarray(full_shape, dtype=float)
filt = BryanFilter(coarse_shape, fine_shape, fovea_shape)
# print(filt.coarse_shape)
# print(filt.fine_shape)
# print(filt.fovea_shape)
# filt.fovea_margin = fovea_margin
print('estimating error with ' + str(fovea_down_factor) + ' ' + str(fovea_iters) + ' ' + str(fovea_shape))
fig = plt.figure(1)
fig.clf()
ax_disp = plt.gca()
#TODO: resolution set here
plot_disp = ax_disp.imshow(high0, vmin=0, vmax=n_disp)
filt_error = []
for i in range(n_frames):
frame = video[i]
coarse_disp = coarse_bp(frame, down_factor=coarse_down_factor, iters=coarse_iters, **coarse_params)
#TODO: extract method
ratio = np.round(float(frame.shape[1]) / float(coarse_disp.shape[0]))
ij0 = np.round(np.asarray(fovea_ij) / ratio)
coarse_shape = np.round(np.asarray(fovea_shape) / ratio)
ij1 = ij0 + coarse_shape
coarse_subwindow = coarse_disp[ij0[0]:ij1[0], ij0[1]:ij1[1]]
seed = cv2.resize(coarse_subwindow, fovea_shape[::-1])
band_size = 5
seed[band_size:-band_size, band_size:-band_size] = 0
fovea_disp = fovea_bp(frame, fovea_ij, fovea_shape, seed, down_factor=fovea_down_factor, iters=fovea_iters, **fovea_params)
# fovea_disp[:] = 0
filt.compute(positions[i], coarse_disp, fovea_disp, fovea_ij, disp2imu, imu2disp)
# --- update fovea position
if 1:
fm, fn = filt.fovea_shape
icost = cv2.integral(filt.cost)
fcost = -np.inf * np.ones_like(icost)
fcostr = fcost[:-fm, :-fn]
fcostr[:] = icost[fm:, fn:]
fcostr -= icost[:-fm, fn:]
fcostr -= icost[fm:, :-fn]
fcostr += icost[:-fm, :-fn]
fcostr[:, :n_disp] = -np.inf # need space left of fovea for disparity
#TODO: set fovea_ij here (currently trying to figure out why errors don't depend on params without setting it)
fovea_ij = np.unravel_index(np.argmax(fcost), fcost.shape)
# translate from fine_shape into full_shape coordinates
fovea_ij = np.round(np.asarray(fovea_ij) / fine_full_ratio)
xyd = load_disparity_points(drive, i)
err = error_on_points(xyd, filt.disp, n_disp, kind='close')
filt_error.append(err)
# show results
img = cv2.cvtColor(frame[0], cv2.COLOR_GRAY2RGB)
# Point = lambda ij: (int(ij[1]), int(ij[0]))
# draw_rect = lambda img, ij, shape, color: cv2.rectangle(
# img, Point(ij), Point((ij[0] + shape[0], ij[1] + shape[1])), color, thickness=2)
# draw_rect(img, fovea_ij, fovea_shape, (255, 0, 0))
# draw_rect(img, new_fovea_ij, fovea_shape, (0, 255, 0))
plot_disp.set_data(filt.disp)
fig.canvas.draw()
######
print('error ' + str(filt_error))
return np.mean(filt_error)
from hunse_tools.timing import tic, toc
idrive = 51
iframe = 50
frame = load_stereo_frame(idrive, iframe)
points = load_disparity_points(idrive, iframe)
n_disp = 128
down_factor = 2
params = {
'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425,
'data_max': 32.024780646200725, 'ksize': 3}
tic()
disp = coarse_bp(frame, values=n_disp, down_factor=down_factor, **params)
toc()
error = error_on_points(points, disp, kind='abs')
print("Error: %f" % error)
# plt.ion()
plt.figure(1)
plt.clf()
axes = [plt.subplot(3, 1, i+1) for i in range(3)]
frame = (downsample(frame[0], down_factor), downsample(frame[1], down_factor))
axes[0].imshow(frame[0], cmap='gray', interpolation='none')
img0 = laplacian(frame[0], ksize=params['ksize'])
axes[1].imshow(img0, cmap='gray', interpolation='none')
axes[2].imshow(disp, vmin=0, vmax=n_disp, interpolation='none')
plt.show()
def process_frame(self, frame, cost=None):
start_time = time.time()
if cost is None:
# Use coarse BP run to get importance for current frame
# params = {
# 'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425,
# 'data_max': 32.024780646200725, 'laplacian_ksize': 3}
prior_params = {
'data_exp': 1.09821084614,
'data_max': 112.191597317,
'data_weight': 0.0139569211273,
'disc_max': 12.1301410452,
'laplacian_ksize': 3,
'smooth': 1.84510833504e-07
}
prior_disparity = coarse_bp(frame,
down_factor=self.mem_down_factor,
iters=5,
values=self.values,
**prior_params)
prior_disparity *= self.frame_step
prior_disparity = prior_disparity[:, self.values / self.mem_step:]
cost = self._uc.get_importance(prior_disparity)
assert cost.shape == self.average_disparity.shape
# c) Find region of highest cost and put fovea there
# mem_fovea_shape = np.array(self.fovea_shape) / self.mem_step
# fovea_corner = _choose_fovea(cost, mem_fovea_shape, 0)
# fovea_corner = np.array(fovea_corner) * self.mem_step + np.array([0, self.values])
mem_fovea_shape = np.array(self.fovea_shape) / self.mem_step
fovea_corners, mem_fovea_shape = _choose_foveas(
cost, mem_fovea_shape, self.values / self.mem_step,
self.max_n_foveas)
# ### debug plot
# print(fovea_corners)
# print(mem_fovea_shape)
# plt.imshow(cost, vmin=0, vmax=128/self.mem_step)
# for (fi, fj) in fovea_corners:
# fm, fn = mem_fovea_shape
# plt.plot([fj, fj+fn, fj+fn, fj, fj], [fi, fi, fi+fm, fi+fm, fi], 'white')
# plt.colorbar()
# plt.show()
# ###
# rescale shape and corners and trim fovea to image ...
fovea_shape = np.array(
mem_fovea_shape
) * self.mem_step # this isn't redundant because _choose_foveas chooses the multifovea shape
fovea_corners = np.array(fovea_corners, dtype='int32')
for i in range(len(fovea_corners)):
fovea_corners[i] = np.array(
fovea_corners[i]) * self.mem_step + np.array([0, self.values])
# fovea corner can be slightly too large, due to rounding errors
fovea_max = np.array(self.frame_shape) - fovea_shape
fovea_corners[i] = np.minimum(fovea_corners[i], fovea_max)
assert fovea_corners[i][0] >= 0 and fovea_corners[i][
1] >= self.values
assert all(fovea_corners[i] + fovea_shape <= self.frame_shape)
# --- fovea boundaries in frame coordinates ...
bp_time = time.time()
disp = foveal_bp(frame,
fovea_corners,
fovea_shape,
values=self.values,
post_smooth=self.post_smooth,
**self.params)
disp *= self.frame_step
self.bp_time = time.time() - bp_time
return disp, fovea_corners
def calc_ground_truth(frame, n_disp, down_factor=0, iters=50):
params = {'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425, 'data_max': 32.024780646200725, 'laplacian_ksize': 1}
gt = coarse_bp(frame, down_factor=down_factor, iters=iters, values=n_disp, **params)
n_coarse_disp = n_disp / 2**down_factor
gt = gt[:,n_coarse_disp:]
return gt
def process_frame(self, pos, frame):
start_time = time.time()
self.disparity_memory.move(pos)
if self.n_past_fovea > 0:
self.fovea_memory.move(pos)
if self.use_uncertainty:
self.uncertainty_memory.move(pos)
if self.verbose:
print('move time: ' + str(time.time() - start_time))
# 1. Decide where to put fovea and move it there:
if self.disparity_memory.n > 0 and len(self.disparity_memory.transforms) == 0:
fovea_corner = (
np.array(self.frame_shape) - np.array(self.fovea_shape)) / 2
assert all(fovea_corner >= 0)
else:
# a) Transform disparity from previous frame and calculate importance
if self.disparity_memory.n > 0:
prior_disparity = self.disparity_memory.transforms[0] #in current frame coords
else:
# TODO: we don't have GPS for multiview, so we're temporarily replacing past estimate with coarse
# estimate from current frame
params = {
'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425,
'data_max': 32.024780646200725, 'laplacian_ksize': 3}
prior_disparity = coarse_bp(frame, down_factor=self.mem_down_factor, iters=5, values=self.values, **params)
prior_disparity *= self.frame_step
prior_disparity = prior_disparity[:,self.values/self.mem_step:]
importance = self._uc.get_importance(prior_disparity)
# b) Transform uncertainty from previous frame and multiply by importance
if self.use_uncertainty:
uncertainty = np.ones_like(importance)
if len(self.uncertainty_memory.transforms) > 0:
uncertainty = self.uncertainty_memory.transforms[0]
cost = importance * uncertainty
else:
cost = importance
assert cost.shape == self.memory_shape
# c) Find region of highest cost and put fovea there
# mem_fovea_shape = np.array(self.fovea_shape) / self.mem_step
# fovea_corner = _choose_fovea(cost, mem_fovea_shape, 0)
# fovea_corner = np.array(fovea_corner) * self.mem_step + np.array([0, self.values])
mem_fovea_shape = np.array(self.fovea_shape) / self.mem_step
fovea_corners, mem_fovea_shape = _choose_foveas(
cost, mem_fovea_shape, self.values/self.mem_step, self.max_n_foveas)
# ### debug plot
# print(fovea_corners)
# print(mem_fovea_shape)
# plt.imshow(cost, vmin=0, vmax=128/self.mem_step)
# for (fi, fj) in fovea_corners:
# fm, fn = mem_fovea_shape
# plt.plot([fj, fj+fn, fj+fn, fj, fj], [fi, fi, fi+fm, fi+fm, fi], 'white')
# plt.colorbar()
# plt.show()
# ###
# rescale shape and corners and trim fovea to image ...
fovea_shape = np.array(mem_fovea_shape) * self.mem_step # this isn't redundant because _choose_foveas chooses the multifovea shape
fovea_corners = np.array(fovea_corners, dtype='int32')
for i in range(len(fovea_corners)):
fovea_corners[i] = np.array(fovea_corners[i]) * self.mem_step + np.array([0, self.values])
# fovea corner can be slightly too large, due to rounding errors
fovea_max = np.array(self.frame_shape) - fovea_shape
fovea_corners[i] = np.minimum(fovea_corners[i], fovea_max)
assert fovea_corners[i][0] >= 0 and fovea_corners[i][1] >= self.values
assert all(fovea_corners[i] + fovea_shape <= self.frame_shape)
if self.verbose:
print('choose time: ' + str(time.time() - start_time))
# 2. Calculate disparity and store in memory:
if len(self.fovea_memory.transforms) == 0:
seed = np.zeros((0,0), dtype='uint8')
else:
seed = np.zeros(self.frame_shape, dtype='uint8')
for t in self.fovea_memory.transforms:
seed += t
if self.verbose:
print('seed time: ' + str(time.time() - start_time))
# --- fovea boundaries in frame coordinates ...
bp_time = time.time()
disp = foveal_bp(
frame, fovea_corners, fovea_shape, seed,
values=self.values, **self.params)
self.bp_time = time.time() - bp_time
# disp = coarse_bp(frame, down_factor=1, iters=3, values=self.values, **self.params)
# disp = cv2.pyrUp(disp)[:self.frame_shape[0], :self.frame_shape[1]]
# keep all disparities in full image coordinates
disp *= self.frame_step
if self.verbose:
print('BP time: ' + str(time.time() - start_time))
# --- downsample and remember disparity
downsampled = downsample(disp[:,self.values:], self.mem_down_factor)
assert downsampled.shape == self.memory_shape
self.disparity_memory.remember(pos, downsampled)
if self.n_past_fovea > 0:
self.fovea_memory.remember(pos, disp, fovea_corner=fovea_corner)
# 3. Calculate uncertainty and store in memory
if self.use_uncertainty and len(self.disparity_memory.transforms) > 0:
prior_disparity = self.disparity_memory.transforms[0]
uncertainty = np.abs(downsampled - prior_disparity)
self.uncertainty_memory.remember(pos, uncertainty)
if self.verbose:
print('finish time: ' + str(time.time() - start_time))
#TODO: use multiple fovea_corners in history
return disp, fovea_corners
for i in range(source.n_frames):
print(i)
frame = [downsample(source.video[i][0], frame_down_factor),
downsample(source.video[i][1], frame_down_factor)]
true_disp = downsample(source.ground_truth[i], frame_down_factor)
true_points = source.true_points[i]
# --- coarse
if run_coarse:
coarse_params = {
'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425,
'data_max': 32.024780646200725, 'laplacian_ksize': 3}
coarse_time = time.time()
coarse_disp = coarse_bp(frame, down_factor=1, iters=3, values=values, **coarse_params)
coarse_disp = cv2.pyrUp(coarse_disp)[:frame_shape[0],:frame_shape[1]]
coarse_disp *= 2
coarse_time = time.time() - coarse_time
append_table('coarse', coarse_disp[:,values:], true_disp, true_points)
times['coarse'].append(coarse_time)
# --- fine
if run_fine:
fine_params = {
'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425,
'data_max': 32.024780646200725, 'laplacian_ksize': 3, 'data_exp': 1.}
# fine_params = {'iters': 3, 'laplacian_ksize': 5, 'data_weight': 0.0002005996175, 'data_max': 109.330237051, 'data_exp': 6.79481234475, 'disc_max': 78.4595739304}
ref_disps = load_fine(drive)
initial = video[0]
n_disp = 128
fovea_shape = (180, 270)
# fovea_shape = (240, 340)
fovea_margin = (10, 10)
fovea_ij = 100, 600
# coarse_params = {'data_weight': 0.0022772439667686963, 'disc_max': 32.257230318943577, 'data_max': 562.30974917928859, 'ksize': 9}
# fovea_params = {'data_weight': 0.058193337153214737, 'disc_max': 5.6647116544500316, 'data_max': 896.07731233120751, 'ksize': 1}
coarse_params = {'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425, 'data_max': 32.024780646200725, 'ksize': 1}
fovea_params = {'data_weight': 0.0095157236661190022, 'disc_max': 1693.5638268258676, 'data_max': 419.34182241802188, 'ksize': 9}
low0 = coarse_bp(initial)
high0 = fovea_bp(initial, fovea_ij, fovea_shape, low0)
high1 = np.zeros(full_shape, dtype=int)
# high1 = fine_bp(initial)
coarse_shape = low0.shape
# filt = RunningAverageFilter(bp_disp0.shape)
# filt = CloseFilter(bp_disp0.shape)
filt = BryanFilter(coarse_shape, fovea_shape)
filt.fovea_margin = fovea_margin
filt_nofovea = BryanFilter(coarse_shape, fovea_shape)
fig = plt.figure(1)
fig.clf()
#raw_input("Place figure...")
'laplacian_ksize': 3,
'smooth': 1.84510833504e-07
}
# args = {'data_exp': 1.09821084614, 'data_max': 112.191597317,
# 'data_weight': 0.0139569211273, 'disc_max': 12.1301410452,
# 'laplacian_ksize': 3, 'smooth': 1.84510833504e-07}
iters = 3
index_errors = []
index_costs = []
for down_factor in down_factors:
values = full_values / 2**down_factor
disp = coarse_bp(frame_ten,
values=full_values,
down_factor=down_factor,
iters=iters,
**args)
# compute cost on small disparity
cost = cost_on_points(disp[:, values:],
true_points,
full_shape=full_shape)
index_costs.append(cost)
# rescale and compute cost on full disparity
disp = upsample(disp, down_factor, frame_shape)
edge = int(round(2.5**down_factor))
error_points = errors_on_full_points(disp, true_points, edge=edge)
index_errors.append(error_points[:, 2].mean())
points = load_disparity_points(idrive, iframe)
full_values = 128
frame_down_factor = 1
values = full_values / 2**frame_down_factor
iters = 3
params = {
'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425,
'data_max': 32.024780646200725, 'ksize': 3}
frame = (downsample(frame[0], frame_down_factor),
downsample(frame[1], frame_down_factor))
tic()
coarse_disp = coarse_bp(frame, values=values, down_factor=1, iters=iters, **params)
coarse_disp *= 2**frame_down_factor
toc()
tic()
fine_disp = coarse_bp(frame, values=values, down_factor=0, iters=iters, **params)
fine_disp *= 2**frame_down_factor
toc()
# fovea_corners = (60, 360)
# fovea_shapes = (80, 80)
fovea_corners = [(60, 160), (60, 360)]
fovea_shapes = [(80, 80), (80, 80)]
fovea_corners = np.array(fovea_corners, ndmin=2)
fovea_shapes = np.array(fovea_shapes, ndmin=2)
coarse_down_factor, coarse_iters = find_coarse_params(coarse_time_budget, video, n_disp, drive, **coarse_params)
# coarse_down_factor = 3
# coarse_iters = 2 #TODO
print('best down_factor ' + str(coarse_down_factor) + ' best iters ' + str(coarse_iters))
# coarse_down_factor = 3
# coarse_iters = find_coarse_iters(coarse_down_factor, initial, coarse_time_budget)
# fovea_shape = (180, 270)
fovea_time_budget = time_budget - coarse_time_budget
fovea_down_factor, fovea_iters, fovea_shape = find_fovea_params(fovea_time_budget, video, positions, drive, coarse_down_factor, coarse_iters)
# low0 = coarse_bp(initial, down_factor=3, iters=5)
low0 = coarse_bp(initial, down_factor=coarse_down_factor, iters=coarse_iters)
# high0 = fovea_bp(initial, fovea_ij, fovea_shape, low0, down_factor=fovea_down_factor, iters=fovea_iters)
high0 = coarse_bp(initial, down_factor=fovea_down_factor, iters=fovea_iters)
high1 = np.zeros(full_shape, dtype=int)
# high1 = fine_bp(initial)
coarse_shape = low0.shape
fine_shape = np.asarray(high0.shape)
filt = BryanFilter(coarse_shape, fine_shape, fovea_shape)
# filt.fovea_margin = fovea_margin
# filt_nofovea = BryanFilter(coarse_shape, fovea_shape)
fig = plt.figure(1)
fig.clf()
