def objective(args): error = 0.0 for source, frame_ten, true_points in sources: full_shape = source.frame_ten[0].shape frame_shape = frame_ten[0].shape frame_ten = [downsample(source.frame_ten[0], frame_down_factor), downsample(source.frame_ten[1], frame_down_factor)] # --- BP disp = coarse_bp( frame_ten, values=values, down_factor=0, iters=iters, **args) disp *= 2**frame_down_factor # --- compute error cost_i = cost_on_points(disp[:,values:], true_points, full_shape=full_shape) error += cost_i if 0: print(cost_i, error) plt.figure() rows, cols = 2, 1 plt.subplot(rows, cols, 1) plt.imshow(frame_ten[0], cmap='gray') plt.subplot(rows, cols, 2) plt.imshow(disp, vmin=0, vmax=64) # plt.imshow(disp, vmin=0, vmax=full_values) plt.show() error /= len(sources) errors[arg_key(args)] = error args_s = "{%s}" % ', '.join("%r: %s" % (k, args[k]) for k in sorted(space)) print("%s: %s" % (error, args_s)) return error
def eval_coarse(frame_ten, frame_shape, values=values): args = { 'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425, 'data_max': 32.024780646200725, 'laplacian_ksize': 3} coarse_time = time.time() coarse_disp = coarse_bp(frame_ten, values=values, down_factor=1, iters=iters, **args) coarse_disp = cv2.pyrUp(coarse_disp)[:frame_shape[0],:frame_shape[1]] coarse_disp *= 2**frame_down_factor coarse_time = time.time() - coarse_time return coarse_disp, coarse_time
def estimate_coarse_error(down_factor, iters, video, n_disp, drive, n_frames=20, **params): # TODO: use fine BP as ground truth? coarse_error = [] for i in range(n_frames): print(i) frame = video[i] tic('coarse') coarse_disp = coarse_bp(frame, down_factor=down_factor, iters=iters, **params) toc() xyd = load_disparity_points(drive, i) coarse_error.append(error_on_points(xyd, coarse_disp, n_disp, kind='close')) return np.mean(coarse_error)
def _evaluate_frame(frame, true_points, frame_shape, full_shape, down_factor, frame_down_factor, iters, params): values = 128/2**frame_down_factor start_time = time.time() coarse_disp = coarse_bp(frame, down_factor=down_factor, iters=iters, values=128, **params) disp = expand_coarse(coarse_disp, down_factor - frame_down_factor) disp = disp[:frame_shape[0],:frame_shape[1]] elapsed_time = time.time() - start_time # unweighted_cost.append(cost_on_points(coarse_disp[:, values:], true_points, full_shape=full_shape)) # true_points = source.true_points[frame_num] return elapsed_time, cost_on_points(disp[:,values:], true_points, full_shape=full_shape)
def calc_ground_truth(frame, n_disp, down_factor=0, iters=50): params = { 'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425, 'data_max': 32.024780646200725, 'laplacian_ksize': 1 } gt = coarse_bp(frame, down_factor=down_factor, iters=iters, values=n_disp, **params) n_coarse_disp = n_disp / 2**down_factor gt = gt[:, n_coarse_disp:] return gt
def eval_fine(frame_ten, values=values): # args = { # 'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425, # 'data_max': 32.024780646200725, 'laplacian_ksize': 3} args = {'data_exp': 1.09821084614, 'data_max': 112.191597317, 'data_weight': 0.0139569211273, 'disc_max': 12.1301410452, 'laplacian_ksize': 3, 'smooth': 1.84510833504e-07} # args = { # 'data_exp': 0.697331027724, 'data_max': 49046.1429149, # 'data_weight': 0.015291472002, 'disc_max': 599730.981833, # 'laplacian_ksize': 5, 'laplacian_scale': 0.103881528095, # 'post_smooth': 2.31047577912, 'smooth': 1.30343003387e-05} fine_time = time.time() fine_disp = coarse_bp(frame_ten, values=values, down_factor=0, iters=iters, **args) fine_disp *= 2**frame_down_factor fine_time = time.time() - fine_time return fine_disp, fine_time
def _evaluate_frame(frame, true_points, frame_shape, full_shape, down_factor, frame_down_factor, iters, params): values = 128 / 2**frame_down_factor start_time = time.time() coarse_disp = coarse_bp(frame, down_factor=down_factor, iters=iters, values=128, **params) disp = expand_coarse(coarse_disp, down_factor - frame_down_factor) disp = disp[:frame_shape[0], :frame_shape[1]] elapsed_time = time.time() - start_time # unweighted_cost.append(cost_on_points(coarse_disp[:, values:], true_points, full_shape=full_shape)) # true_points = source.true_points[frame_num] return elapsed_time, cost_on_points(disp[:, values:], true_points, full_shape=full_shape)
def objective(args): error = 0.0 for source, frame_ten, true_points in sources: full_shape = source.frame_ten[0].shape frame_shape = frame_ten[0].shape frame_ten = [ downsample(source.frame_ten[0], frame_down_factor), downsample(source.frame_ten[1], frame_down_factor) ] # --- BP disp = coarse_bp(frame_ten, values=values, down_factor=0, iters=iters, **args) disp *= 2**frame_down_factor # --- compute error cost_i = cost_on_points(disp[:, values:], true_points, full_shape=full_shape) error += cost_i if 0: print(cost_i, error) plt.figure() rows, cols = 2, 1 plt.subplot(rows, cols, 1) plt.imshow(frame_ten[0], cmap='gray') plt.subplot(rows, cols, 2) plt.imshow(disp, vmin=0, vmax=64) # plt.imshow(disp, vmin=0, vmax=full_values) plt.show() error /= len(sources) errors[arg_key(args)] = error args_s = "{%s}" % ', '.join("%r: %s" % (k, args[k]) for k in sorted(space)) print("%s: %s" % (error, args_s)) return error
iters = 3 params = { 'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425, 'data_max': 32.024780646200725, 'ksize': 3 } frame = (downsample(frame[0], frame_down_factor), downsample(frame[1], frame_down_factor)) tic() coarse_disp = coarse_bp(frame, values=values, down_factor=1, iters=iters, **params) coarse_disp *= 2**frame_down_factor toc() tic() fine_disp = coarse_bp(frame, values=values, down_factor=0, iters=iters, **params) fine_disp *= 2**frame_down_factor toc() # fovea_corners = (60, 360)
def process_frame(self, pos, frame): start_time = time.time() self.disparity_memory.move(pos) if self.n_past_fovea > 0: self.fovea_memory.move(pos) if self.use_uncertainty: self.uncertainty_memory.move(pos) if self.verbose: print('move time: ' + str(time.time() - start_time)) # 1. Decide where to put fovea and move it there: if self.disparity_memory.n > 0 and len( self.disparity_memory.transforms) == 0: fovea_corner = (np.array(self.frame_shape) - np.array(self.fovea_shape)) / 2 assert all(fovea_corner >= 0) else: # a) Transform disparity from previous frame and calculate importance if self.disparity_memory.n > 0: prior_disparity = self.disparity_memory.transforms[ 0] #in current frame coords else: # TODO: we don't have GPS for multiview, so we're temporarily replacing past estimate with coarse # estimate from current frame params = { 'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425, 'data_max': 32.024780646200725, 'laplacian_ksize': 3 } prior_disparity = coarse_bp(frame, down_factor=self.mem_down_factor, iters=5, values=self.values, **params) prior_disparity *= self.frame_step prior_disparity = prior_disparity[:, self.values / self.mem_step:] importance = self._uc.get_importance(prior_disparity) # b) Transform uncertainty from previous frame and multiply by importance if self.use_uncertainty: uncertainty = np.ones_like(importance) if len(self.uncertainty_memory.transforms) > 0: uncertainty = self.uncertainty_memory.transforms[0] cost = importance * uncertainty else: cost = importance assert cost.shape == self.memory_shape # c) Find region of highest cost and put fovea there # mem_fovea_shape = np.array(self.fovea_shape) / self.mem_step # fovea_corner = _choose_fovea(cost, mem_fovea_shape, 0) # fovea_corner = np.array(fovea_corner) * self.mem_step + np.array([0, self.values]) mem_fovea_shape = np.array(self.fovea_shape) / self.mem_step fovea_corners, mem_fovea_shape = _choose_foveas( cost, mem_fovea_shape, self.values / self.mem_step, self.max_n_foveas) # ### debug plot # print(fovea_corners) # print(mem_fovea_shape) # plt.imshow(cost, vmin=0, vmax=128/self.mem_step) # for (fi, fj) in fovea_corners: # fm, fn = mem_fovea_shape # plt.plot([fj, fj+fn, fj+fn, fj, fj], [fi, fi, fi+fm, fi+fm, fi], 'white') # plt.colorbar() # plt.show() # ### # rescale shape and corners and trim fovea to image ... fovea_shape = np.array( mem_fovea_shape ) * self.mem_step # this isn't redundant because _choose_foveas chooses the multifovea shape fovea_corners = np.array(fovea_corners, dtype='int32') for i in range(len(fovea_corners)): fovea_corners[i] = np.array( fovea_corners[i]) * self.mem_step + np.array( [0, self.values]) # fovea corner can be slightly too large, due to rounding errors fovea_max = np.array(self.frame_shape) - fovea_shape fovea_corners[i] = np.minimum(fovea_corners[i], fovea_max) assert fovea_corners[i][0] >= 0 and fovea_corners[i][ 1] >= self.values assert all(fovea_corners[i] + fovea_shape <= self.frame_shape) if self.verbose: print('choose time: ' + str(time.time() - start_time)) # 2. Calculate disparity and store in memory: if len(self.fovea_memory.transforms) == 0: seed = np.zeros((0, 0), dtype='uint8') else: seed = np.zeros(self.frame_shape, dtype='uint8') for t in self.fovea_memory.transforms: seed += t if self.verbose: print('seed time: ' + str(time.time() - start_time)) # --- fovea boundaries in frame coordinates ... bp_time = time.time() disp = foveal_bp(frame, fovea_corners, fovea_shape, seed, values=self.values, **self.params) self.bp_time = time.time() - bp_time # disp = coarse_bp(frame, down_factor=1, iters=3, values=self.values, **self.params) # disp = cv2.pyrUp(disp)[:self.frame_shape[0], :self.frame_shape[1]] # keep all disparities in full image coordinates disp *= self.frame_step if self.verbose: print('BP time: ' + str(time.time() - start_time)) # --- downsample and remember disparity downsampled = downsample(disp[:, self.values:], self.mem_down_factor) assert downsampled.shape == self.memory_shape self.disparity_memory.remember(pos, downsampled) if self.n_past_fovea > 0: self.fovea_memory.remember(pos, disp, fovea_corner=fovea_corner) # 3. Calculate uncertainty and store in memory if self.use_uncertainty and len(self.disparity_memory.transforms) > 0: prior_disparity = self.disparity_memory.transforms[0] uncertainty = np.abs(downsampled - prior_disparity) self.uncertainty_memory.remember(pos, uncertainty) if self.verbose: print('finish time: ' + str(time.time() - start_time)) #TODO: use multiple fovea_corners in history return disp, fovea_corners
] true_disp = downsample(source.ground_truth[i], frame_down_factor) true_points = source.true_points[i] # --- coarse if run_coarse: coarse_params = { 'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425, 'data_max': 32.024780646200725, 'laplacian_ksize': 3 } coarse_time = time.time() coarse_disp = coarse_bp(frame, down_factor=1, iters=3, values=values, **coarse_params) coarse_disp = cv2.pyrUp( coarse_disp)[:frame_shape[0], :frame_shape[1]] coarse_disp *= 2 coarse_time = time.time() - coarse_time append_table('coarse', coarse_disp[:, values:], true_disp, true_points) times['coarse'].append(coarse_time) # --- fine if run_fine: fine_params = { 'data_weight': 0.16145115747533928,
iframe = 50 frame = load_stereo_frame(idrive, iframe) points = load_disparity_points(idrive, iframe) n_disp = 128 down_factor = 2 params = { 'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425, 'data_max': 32.024780646200725, 'ksize': 3 } tic() disp = coarse_bp(frame, values=n_disp, down_factor=down_factor, **params) toc() error = error_on_points(points, disp, kind='abs') print("Error: %f" % error) # plt.ion() plt.figure(1) plt.clf() axes = [plt.subplot(3, 1, i + 1) for i in range(3)] frame = (downsample(frame[0], down_factor), downsample(frame[1], down_factor)) axes[0].imshow(frame[0], cmap='gray', interpolation='none') img0 = laplacian(frame[0], ksize=params['ksize']) axes[1].imshow(img0, cmap='gray', interpolation='none') axes[2].imshow(disp, vmin=0, vmax=n_disp, interpolation='none') plt.show()
def time_coarse(down_factor, video, iters): before = time.time() n_frames = 3 for i in range(n_frames): coarse_bp(video[i], down_factor=down_factor, iters=iters) return (time.time() - before) / n_frames
def estimate_fovea_error(coarse_down_factor, coarse_iters, fovea_down_factor, fovea_iters, fovea_shape, video, positions, drive, n_frames=20): fovea_ij = 200, 600 #note: ground truth only available in lower part of frame #fovea_ij = 40, 100 low0 = coarse_bp(video[0], down_factor=coarse_down_factor, iters=coarse_iters) high0 = coarse_bp(video[0], down_factor=fovea_down_factor, iters=fovea_iters) high1 = fovea_bp(video[0], fovea_ij, fovea_shape, low0, down_factor=fovea_down_factor, iters=fovea_iters) # high1 = np.zeros(full_shape, dtype=int) # high1 = fine_bp(initial) coarse_shape = np.asarray(low0.shape) fine_shape = np.asarray(high0.shape) # fine_shape = full_shape fine_full_ratio = fine_shape / np.asarray(full_shape, dtype=float) filt = BryanFilter(coarse_shape, fine_shape, fovea_shape) # print(filt.coarse_shape) # print(filt.fine_shape) # print(filt.fovea_shape) # filt.fovea_margin = fovea_margin print('estimating error with ' + str(fovea_down_factor) + ' ' + str(fovea_iters) + ' ' + str(fovea_shape)) fig = plt.figure(1) fig.clf() ax_disp = plt.gca() #TODO: resolution set here plot_disp = ax_disp.imshow(high0, vmin=0, vmax=n_disp) filt_error = [] for i in range(n_frames): frame = video[i] coarse_disp = coarse_bp(frame, down_factor=coarse_down_factor, iters=coarse_iters, **coarse_params) #TODO: extract method ratio = np.round(float(frame.shape[1]) / float(coarse_disp.shape[0])) ij0 = np.round(np.asarray(fovea_ij) / ratio) coarse_shape = np.round(np.asarray(fovea_shape) / ratio) ij1 = ij0 + coarse_shape coarse_subwindow = coarse_disp[ij0[0]:ij1[0], ij0[1]:ij1[1]] seed = cv2.resize(coarse_subwindow, fovea_shape[::-1]) band_size = 5 seed[band_size:-band_size, band_size:-band_size] = 0 fovea_disp = fovea_bp(frame, fovea_ij, fovea_shape, seed, down_factor=fovea_down_factor, iters=fovea_iters, **fovea_params) # fovea_disp[:] = 0 filt.compute(positions[i], coarse_disp, fovea_disp, fovea_ij, disp2imu, imu2disp) # --- update fovea position if 1: fm, fn = filt.fovea_shape icost = cv2.integral(filt.cost) fcost = -np.inf * np.ones_like(icost) fcostr = fcost[:-fm, :-fn] fcostr[:] = icost[fm:, fn:] fcostr -= icost[:-fm, fn:] fcostr -= icost[fm:, :-fn] fcostr += icost[:-fm, :-fn] fcostr[:, :n_disp] = -np.inf # need space left of fovea for disparity #TODO: set fovea_ij here (currently trying to figure out why errors don't depend on params without setting it) fovea_ij = np.unravel_index(np.argmax(fcost), fcost.shape) # translate from fine_shape into full_shape coordinates fovea_ij = np.round(np.asarray(fovea_ij) / fine_full_ratio) xyd = load_disparity_points(drive, i) err = error_on_points(xyd, filt.disp, n_disp, kind='close') filt_error.append(err) # show results img = cv2.cvtColor(frame[0], cv2.COLOR_GRAY2RGB) # Point = lambda ij: (int(ij[1]), int(ij[0])) # draw_rect = lambda img, ij, shape, color: cv2.rectangle( # img, Point(ij), Point((ij[0] + shape[0], ij[1] + shape[1])), color, thickness=2) # draw_rect(img, fovea_ij, fovea_shape, (255, 0, 0)) # draw_rect(img, new_fovea_ij, fovea_shape, (0, 255, 0)) plot_disp.set_data(filt.disp) fig.canvas.draw() ###### print('error ' + str(filt_error)) return np.mean(filt_error)
from hunse_tools.timing import tic, toc idrive = 51 iframe = 50 frame = load_stereo_frame(idrive, iframe) points = load_disparity_points(idrive, iframe) n_disp = 128 down_factor = 2 params = { 'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425, 'data_max': 32.024780646200725, 'ksize': 3} tic() disp = coarse_bp(frame, values=n_disp, down_factor=down_factor, **params) toc() error = error_on_points(points, disp, kind='abs') print("Error: %f" % error) # plt.ion() plt.figure(1) plt.clf() axes = [plt.subplot(3, 1, i+1) for i in range(3)] frame = (downsample(frame[0], down_factor), downsample(frame[1], down_factor)) axes[0].imshow(frame[0], cmap='gray', interpolation='none') img0 = laplacian(frame[0], ksize=params['ksize']) axes[1].imshow(img0, cmap='gray', interpolation='none') axes[2].imshow(disp, vmin=0, vmax=n_disp, interpolation='none') plt.show()
def process_frame(self, frame, cost=None): start_time = time.time() if cost is None: # Use coarse BP run to get importance for current frame # params = { # 'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425, # 'data_max': 32.024780646200725, 'laplacian_ksize': 3} prior_params = { 'data_exp': 1.09821084614, 'data_max': 112.191597317, 'data_weight': 0.0139569211273, 'disc_max': 12.1301410452, 'laplacian_ksize': 3, 'smooth': 1.84510833504e-07 } prior_disparity = coarse_bp(frame, down_factor=self.mem_down_factor, iters=5, values=self.values, **prior_params) prior_disparity *= self.frame_step prior_disparity = prior_disparity[:, self.values / self.mem_step:] cost = self._uc.get_importance(prior_disparity) assert cost.shape == self.average_disparity.shape # c) Find region of highest cost and put fovea there # mem_fovea_shape = np.array(self.fovea_shape) / self.mem_step # fovea_corner = _choose_fovea(cost, mem_fovea_shape, 0) # fovea_corner = np.array(fovea_corner) * self.mem_step + np.array([0, self.values]) mem_fovea_shape = np.array(self.fovea_shape) / self.mem_step fovea_corners, mem_fovea_shape = _choose_foveas( cost, mem_fovea_shape, self.values / self.mem_step, self.max_n_foveas) # ### debug plot # print(fovea_corners) # print(mem_fovea_shape) # plt.imshow(cost, vmin=0, vmax=128/self.mem_step) # for (fi, fj) in fovea_corners: # fm, fn = mem_fovea_shape # plt.plot([fj, fj+fn, fj+fn, fj, fj], [fi, fi, fi+fm, fi+fm, fi], 'white') # plt.colorbar() # plt.show() # ### # rescale shape and corners and trim fovea to image ... fovea_shape = np.array( mem_fovea_shape ) * self.mem_step # this isn't redundant because _choose_foveas chooses the multifovea shape fovea_corners = np.array(fovea_corners, dtype='int32') for i in range(len(fovea_corners)): fovea_corners[i] = np.array( fovea_corners[i]) * self.mem_step + np.array([0, self.values]) # fovea corner can be slightly too large, due to rounding errors fovea_max = np.array(self.frame_shape) - fovea_shape fovea_corners[i] = np.minimum(fovea_corners[i], fovea_max) assert fovea_corners[i][0] >= 0 and fovea_corners[i][ 1] >= self.values assert all(fovea_corners[i] + fovea_shape <= self.frame_shape) # --- fovea boundaries in frame coordinates ... bp_time = time.time() disp = foveal_bp(frame, fovea_corners, fovea_shape, values=self.values, post_smooth=self.post_smooth, **self.params) disp *= self.frame_step self.bp_time = time.time() - bp_time return disp, fovea_corners
def calc_ground_truth(frame, n_disp, down_factor=0, iters=50): params = {'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425, 'data_max': 32.024780646200725, 'laplacian_ksize': 1} gt = coarse_bp(frame, down_factor=down_factor, iters=iters, values=n_disp, **params) n_coarse_disp = n_disp / 2**down_factor gt = gt[:,n_coarse_disp:] return gt
def process_frame(self, pos, frame): start_time = time.time() self.disparity_memory.move(pos) if self.n_past_fovea > 0: self.fovea_memory.move(pos) if self.use_uncertainty: self.uncertainty_memory.move(pos) if self.verbose: print('move time: ' + str(time.time() - start_time)) # 1. Decide where to put fovea and move it there: if self.disparity_memory.n > 0 and len(self.disparity_memory.transforms) == 0: fovea_corner = ( np.array(self.frame_shape) - np.array(self.fovea_shape)) / 2 assert all(fovea_corner >= 0) else: # a) Transform disparity from previous frame and calculate importance if self.disparity_memory.n > 0: prior_disparity = self.disparity_memory.transforms[0] #in current frame coords else: # TODO: we don't have GPS for multiview, so we're temporarily replacing past estimate with coarse # estimate from current frame params = { 'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425, 'data_max': 32.024780646200725, 'laplacian_ksize': 3} prior_disparity = coarse_bp(frame, down_factor=self.mem_down_factor, iters=5, values=self.values, **params) prior_disparity *= self.frame_step prior_disparity = prior_disparity[:,self.values/self.mem_step:] importance = self._uc.get_importance(prior_disparity) # b) Transform uncertainty from previous frame and multiply by importance if self.use_uncertainty: uncertainty = np.ones_like(importance) if len(self.uncertainty_memory.transforms) > 0: uncertainty = self.uncertainty_memory.transforms[0] cost = importance * uncertainty else: cost = importance assert cost.shape == self.memory_shape # c) Find region of highest cost and put fovea there # mem_fovea_shape = np.array(self.fovea_shape) / self.mem_step # fovea_corner = _choose_fovea(cost, mem_fovea_shape, 0) # fovea_corner = np.array(fovea_corner) * self.mem_step + np.array([0, self.values]) mem_fovea_shape = np.array(self.fovea_shape) / self.mem_step fovea_corners, mem_fovea_shape = _choose_foveas( cost, mem_fovea_shape, self.values/self.mem_step, self.max_n_foveas) # ### debug plot # print(fovea_corners) # print(mem_fovea_shape) # plt.imshow(cost, vmin=0, vmax=128/self.mem_step) # for (fi, fj) in fovea_corners: # fm, fn = mem_fovea_shape # plt.plot([fj, fj+fn, fj+fn, fj, fj], [fi, fi, fi+fm, fi+fm, fi], 'white') # plt.colorbar() # plt.show() # ### # rescale shape and corners and trim fovea to image ... fovea_shape = np.array(mem_fovea_shape) * self.mem_step # this isn't redundant because _choose_foveas chooses the multifovea shape fovea_corners = np.array(fovea_corners, dtype='int32') for i in range(len(fovea_corners)): fovea_corners[i] = np.array(fovea_corners[i]) * self.mem_step + np.array([0, self.values]) # fovea corner can be slightly too large, due to rounding errors fovea_max = np.array(self.frame_shape) - fovea_shape fovea_corners[i] = np.minimum(fovea_corners[i], fovea_max) assert fovea_corners[i][0] >= 0 and fovea_corners[i][1] >= self.values assert all(fovea_corners[i] + fovea_shape <= self.frame_shape) if self.verbose: print('choose time: ' + str(time.time() - start_time)) # 2. Calculate disparity and store in memory: if len(self.fovea_memory.transforms) == 0: seed = np.zeros((0,0), dtype='uint8') else: seed = np.zeros(self.frame_shape, dtype='uint8') for t in self.fovea_memory.transforms: seed += t if self.verbose: print('seed time: ' + str(time.time() - start_time)) # --- fovea boundaries in frame coordinates ... bp_time = time.time() disp = foveal_bp( frame, fovea_corners, fovea_shape, seed, values=self.values, **self.params) self.bp_time = time.time() - bp_time # disp = coarse_bp(frame, down_factor=1, iters=3, values=self.values, **self.params) # disp = cv2.pyrUp(disp)[:self.frame_shape[0], :self.frame_shape[1]] # keep all disparities in full image coordinates disp *= self.frame_step if self.verbose: print('BP time: ' + str(time.time() - start_time)) # --- downsample and remember disparity downsampled = downsample(disp[:,self.values:], self.mem_down_factor) assert downsampled.shape == self.memory_shape self.disparity_memory.remember(pos, downsampled) if self.n_past_fovea > 0: self.fovea_memory.remember(pos, disp, fovea_corner=fovea_corner) # 3. Calculate uncertainty and store in memory if self.use_uncertainty and len(self.disparity_memory.transforms) > 0: prior_disparity = self.disparity_memory.transforms[0] uncertainty = np.abs(downsampled - prior_disparity) self.uncertainty_memory.remember(pos, uncertainty) if self.verbose: print('finish time: ' + str(time.time() - start_time)) #TODO: use multiple fovea_corners in history return disp, fovea_corners
for i in range(source.n_frames): print(i) frame = [downsample(source.video[i][0], frame_down_factor), downsample(source.video[i][1], frame_down_factor)] true_disp = downsample(source.ground_truth[i], frame_down_factor) true_points = source.true_points[i] # --- coarse if run_coarse: coarse_params = { 'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425, 'data_max': 32.024780646200725, 'laplacian_ksize': 3} coarse_time = time.time() coarse_disp = coarse_bp(frame, down_factor=1, iters=3, values=values, **coarse_params) coarse_disp = cv2.pyrUp(coarse_disp)[:frame_shape[0],:frame_shape[1]] coarse_disp *= 2 coarse_time = time.time() - coarse_time append_table('coarse', coarse_disp[:,values:], true_disp, true_points) times['coarse'].append(coarse_time) # --- fine if run_fine: fine_params = { 'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425, 'data_max': 32.024780646200725, 'laplacian_ksize': 3, 'data_exp': 1.} # fine_params = {'iters': 3, 'laplacian_ksize': 5, 'data_weight': 0.0002005996175, 'data_max': 109.330237051, 'data_exp': 6.79481234475, 'disc_max': 78.4595739304}
ref_disps = load_fine(drive) initial = video[0] n_disp = 128 fovea_shape = (180, 270) # fovea_shape = (240, 340) fovea_margin = (10, 10) fovea_ij = 100, 600 # coarse_params = {'data_weight': 0.0022772439667686963, 'disc_max': 32.257230318943577, 'data_max': 562.30974917928859, 'ksize': 9} # fovea_params = {'data_weight': 0.058193337153214737, 'disc_max': 5.6647116544500316, 'data_max': 896.07731233120751, 'ksize': 1} coarse_params = {'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425, 'data_max': 32.024780646200725, 'ksize': 1} fovea_params = {'data_weight': 0.0095157236661190022, 'disc_max': 1693.5638268258676, 'data_max': 419.34182241802188, 'ksize': 9} low0 = coarse_bp(initial) high0 = fovea_bp(initial, fovea_ij, fovea_shape, low0) high1 = np.zeros(full_shape, dtype=int) # high1 = fine_bp(initial) coarse_shape = low0.shape # filt = RunningAverageFilter(bp_disp0.shape) # filt = CloseFilter(bp_disp0.shape) filt = BryanFilter(coarse_shape, fovea_shape) filt.fovea_margin = fovea_margin filt_nofovea = BryanFilter(coarse_shape, fovea_shape) fig = plt.figure(1) fig.clf() #raw_input("Place figure...")
'laplacian_ksize': 3, 'smooth': 1.84510833504e-07 } # args = {'data_exp': 1.09821084614, 'data_max': 112.191597317, # 'data_weight': 0.0139569211273, 'disc_max': 12.1301410452, # 'laplacian_ksize': 3, 'smooth': 1.84510833504e-07} iters = 3 index_errors = [] index_costs = [] for down_factor in down_factors: values = full_values / 2**down_factor disp = coarse_bp(frame_ten, values=full_values, down_factor=down_factor, iters=iters, **args) # compute cost on small disparity cost = cost_on_points(disp[:, values:], true_points, full_shape=full_shape) index_costs.append(cost) # rescale and compute cost on full disparity disp = upsample(disp, down_factor, frame_shape) edge = int(round(2.5**down_factor)) error_points = errors_on_full_points(disp, true_points, edge=edge) index_errors.append(error_points[:, 2].mean())
points = load_disparity_points(idrive, iframe) full_values = 128 frame_down_factor = 1 values = full_values / 2**frame_down_factor iters = 3 params = { 'data_weight': 0.16145115747533928, 'disc_max': 294.1504935618425, 'data_max': 32.024780646200725, 'ksize': 3} frame = (downsample(frame[0], frame_down_factor), downsample(frame[1], frame_down_factor)) tic() coarse_disp = coarse_bp(frame, values=values, down_factor=1, iters=iters, **params) coarse_disp *= 2**frame_down_factor toc() tic() fine_disp = coarse_bp(frame, values=values, down_factor=0, iters=iters, **params) fine_disp *= 2**frame_down_factor toc() # fovea_corners = (60, 360) # fovea_shapes = (80, 80) fovea_corners = [(60, 160), (60, 360)] fovea_shapes = [(80, 80), (80, 80)] fovea_corners = np.array(fovea_corners, ndmin=2) fovea_shapes = np.array(fovea_shapes, ndmin=2)
coarse_down_factor, coarse_iters = find_coarse_params(coarse_time_budget, video, n_disp, drive, **coarse_params) # coarse_down_factor = 3 # coarse_iters = 2 #TODO print('best down_factor ' + str(coarse_down_factor) + ' best iters ' + str(coarse_iters)) # coarse_down_factor = 3 # coarse_iters = find_coarse_iters(coarse_down_factor, initial, coarse_time_budget) # fovea_shape = (180, 270) fovea_time_budget = time_budget - coarse_time_budget fovea_down_factor, fovea_iters, fovea_shape = find_fovea_params(fovea_time_budget, video, positions, drive, coarse_down_factor, coarse_iters) # low0 = coarse_bp(initial, down_factor=3, iters=5) low0 = coarse_bp(initial, down_factor=coarse_down_factor, iters=coarse_iters) # high0 = fovea_bp(initial, fovea_ij, fovea_shape, low0, down_factor=fovea_down_factor, iters=fovea_iters) high0 = coarse_bp(initial, down_factor=fovea_down_factor, iters=fovea_iters) high1 = np.zeros(full_shape, dtype=int) # high1 = fine_bp(initial) coarse_shape = low0.shape fine_shape = np.asarray(high0.shape) filt = BryanFilter(coarse_shape, fine_shape, fovea_shape) # filt.fovea_margin = fovea_margin # filt_nofovea = BryanFilter(coarse_shape, fovea_shape) fig = plt.figure(1) fig.clf()