def register_horizontally_shear(matches, H1, H2): """ Adjust rectifying homographies with tilt, shear and translation to reduce the disparity range. Args: matches: list of pairs of 2D points, stored as a Nx4 numpy array H1, H2: two homographies, stored as numpy 3x3 matrices Returns: H2: corrected homography H2 The matches are provided in the original images coordinate system. By transforming these coordinates with the provided homographies, we obtain matches whose disparity is only along the x-axis. """ # transform the matches according to the homographies p1 = common.points_apply_homography(H1, matches[:, :2]) x1 = p1[:, 0] y1 = p1[:, 1] p2 = common.points_apply_homography(H2, matches[:, 2:]) x2 = p2[:, 0] y2 = p2[:, 1] if cfg['debug']: print("Residual vertical disparities: max, min, mean. Should be zero") print(np.max(y2 - y1), np.min(y2 - y1), np.mean(y2 - y1)) # we search the (a, b, c) vector that minimises \sum (x1 - (a*x2+b*y2+c))^2 # it is a least squares minimisation problem A = np.vstack((x2, y2, y2 * 0 + 1)).T a, b, c = np.linalg.lstsq(A, x1)[0].flatten() # correct H2 with the estimated tilt, shear and translation return np.dot(np.array([[a, b, c], [0, 1, 0], [0, 0, 1]]), H2)
def disparity_range_from_matches(matches, H1, H2, w, h): """ Compute the disparity range of a ROI from a list of point matches. Args: matches: Nx4 numpy array containing a list of matches, in the full image coordinates frame, before rectification w, h: width and height of the rectangular ROI in the first image. H1, H2: two rectifying homographies, stored as numpy 3x3 matrices Returns: disp_min, disp_max: horizontal disparity range """ # transform the matches according to the homographies p1 = common.points_apply_homography(H1, matches[:, :2]) x1 = p1[:, 0] p2 = common.points_apply_homography(H2, matches[:, 2:]) x2 = p2[:, 0] # compute the final disparity range disp_min = np.floor(np.min(x2 - x1)) disp_max = np.ceil(np.max(x2 - x1)) # add a security margin to the disparity range disp_min -= (disp_max - disp_min) * cfg['disp_range_extra_margin'] disp_max += (disp_max - disp_min) * cfg['disp_range_extra_margin'] return disp_min, disp_max
def alt_to_disp(rpc1, rpc2, x, y, alt, H1, H2, A=None): """ Converts an altitude into a disparity. Args: rpc1: an instance of the rpcm.RPCModel class for the reference image rpc2: an instance of the rpcm.RPCModel class for the secondary image x, y: coordinates of the point in the reference image alt: altitude above the WGS84 ellipsoid (in meters) of the point H1, H2: rectifying homographies A (optional): pointing correction matrix Returns: the horizontal disparity of the (x, y) point of im1, assuming that the 3-space point associated has altitude alt. The disparity is made horizontal thanks to the two rectifying homographies H1 and H2. """ xx, yy = find_corresponding_point(rpc1, rpc2, x, y, alt)[0:2] p1 = np.vstack([x, y]).T p2 = np.vstack([xx, yy]).T if A is not None: print("rpc_utils.alt_to_disp: applying pointing error correction") # correct coordinates of points in im2, according to A p2 = common.points_apply_homography(np.linalg.inv(A), p2) p1 = common.points_apply_homography(H1, p1) p2 = common.points_apply_homography(H2, p2) # np.testing.assert_allclose(p1[:, 1], p2[:, 1], atol=0.1) disp = p2[:, 0] - p1[:, 0] return disp
def test_affine_transformation(): x = np.array([[0, 0], [0, 1], [1, 0], [1, 1]]) # list of transformations to be tested T = np.eye(3) I = np.eye(3) S = np.eye(3) A = np.eye(3) translations = [] isometries = [] similarities = [] affinities = [] for i in range(100): translations.append(T) isometries.append(I) similarities.append(S) affinities.append(A) T[0:2, 2] = np.random.random(2) I = rotation_matrix(2 * np.pi * np.random.random_sample()) I[0:2, 2] = np.random.random(2) S = similarity_matrix(2 * np.pi * np.random.random_sample(), np.random.random_sample()) S[0:2, 2] = 100 * np.random.random(2) A[0:2, :] = np.random.random((2, 3)) for B in translations + isometries + similarities + affinities: xx = common.points_apply_homography(B, x) E = estimation.affine_transformation(x, xx) assert_array_almost_equal(E, B)
def register_horizontally_translation(matches, H1, H2, flag='center'): """ Adjust rectifying homographies with a translation to modify the disparity range. Args: matches: list of pairs of 2D points, stored as a Nx4 numpy array H1, H2: two homographies, stored as numpy 3x3 matrices flag: option needed to control how to modify the disparity range: 'center': move the barycenter of disparities of matches to zero 'positive': make all the disparities positive 'negative': make all the disparities negative. Required for Hirshmuller stereo (java) Returns: H2: corrected homography H2 The matches are provided in the original images coordinate system. By transforming these coordinates with the provided homographies, we obtain matches whose disparity is only along the x-axis. The second homography H2 is corrected with a horizontal translation to obtain the desired property on the disparity range. """ # transform the matches according to the homographies p1 = common.points_apply_homography(H1, matches[:, :2]) x1 = p1[:, 0] y1 = p1[:, 1] p2 = common.points_apply_homography(H2, matches[:, 2:]) x2 = p2[:, 0] y2 = p2[:, 1] # for debug, print the vertical disparities. Should be zero. if cfg['debug']: print("Residual vertical disparities: max, min, mean. Should be zero") print(np.max(y2 - y1), np.min(y2 - y1), np.mean(y2 - y1)) # compute the disparity offset according to selected option t = 0 if (flag == 'center'): t = np.mean(x2 - x1) if (flag == 'positive'): t = np.min(x2 - x1) if (flag == 'negative'): t = np.max(x2 - x1) # correct H2 with a translation return np.dot(common.matrix_translation(-t, 0), H2)
def rectification_homographies(matches, x, y, w, h): """ Computes rectifying homographies from point matches for a given ROI. The affine fundamental matrix F is estimated with the gold-standard algorithm, then two rectifying similarities (rotation, zoom, translation) are computed directly from F. Args: matches: numpy array of shape (n, 4) containing a list of 2D point correspondences between the two images. x, y, w, h: four integers defining the rectangular ROI in the first image. (x, y) is the top-left corner, and (w, h) are the dimensions of the rectangle. Returns: S1, S2, F: three numpy arrays of shape (3, 3) representing the two rectifying similarities to be applied to the two images and the corresponding affine fundamental matrix. """ # estimate the affine fundamental matrix with the Gold standard algorithm F = estimation.affine_fundamental_matrix(matches) # compute rectifying similarities S1, S2 = estimation.rectifying_similarities_from_affine_fundamental_matrix( F, cfg['debug']) if cfg['debug']: y1 = common.points_apply_homography(S1, matches[:, :2])[:, 1] y2 = common.points_apply_homography(S2, matches[:, 2:])[:, 1] err = np.abs(y1 - y2) print("max, min, mean rectification error on point matches: ", end=' ') print(np.max(err), np.min(err), np.mean(err)) # pull back top-left corner of the ROI to the origin (plus margin) pts = common.points_apply_homography( S1, [[x, y], [x + w, y], [x + w, y + h], [x, y + h]]) x0, y0 = common.bounding_box2D(pts)[:2] T = common.matrix_translation(-x0, -y0) return np.dot(T, S1), np.dot(T, S2), F
def rectify_pair(im1, im2, rpc1, rpc2, x, y, w, h, out1, out2, A=None, sift_matches=None, method='rpc', hmargin=0, vmargin=0): """ Rectify a ROI in a pair of images. Args: im1, im2: paths to two GeoTIFF image files rpc1, rpc2: two instances of the rpcm.RPCModel class x, y, w, h: four integers defining the rectangular ROI in the first image. (x, y) is the top-left corner, and (w, h) are the dimensions of the rectangle. out1, out2: paths to the output rectified crops A (optional): 3x3 numpy array containing the pointing error correction for im2. This matrix is usually estimated with the pointing_accuracy module. sift_matches (optional): Nx4 numpy array containing a list of sift matches, in the full image coordinates frame method (default: 'rpc'): option to decide wether to use rpc of sift matches for the fundamental matrix estimation. {h,v}margin (optional): horizontal and vertical margins added on the sides of the rectified images Returns: H1, H2: Two 3x3 matrices representing the rectifying homographies that have been applied to the two original (large) images. disp_min, disp_max: horizontal disparity range """ # compute real or virtual matches if method == 'rpc': # find virtual matches from RPC camera models matches = rpc_utils.matches_from_rpc(rpc1, rpc2, x, y, w, h, cfg['n_gcp_per_axis']) # correct second image coordinates with the pointing correction matrix if A is not None: matches[:, 2:] = common.points_apply_homography( np.linalg.inv(A), matches[:, 2:]) elif method == 'sift': matches = sift_matches else: raise Exception( "Unknown value {} for argument 'method'".format(method)) if matches is None or len(matches) < 4: raise NoRectificationMatchesError( "No or not enough matches found to rectify image pair") # compute rectifying homographies H1, H2, F = rectification_homographies(matches, x, y, w, h) if cfg['register_with_shear']: # compose H2 with a horizontal shear to reduce the disparity range a = np.mean(rpc_utils.altitude_range(rpc1, x, y, w, h)) lon, lat, alt = rpc_utils.ground_control_points( rpc1, x, y, w, h, a, a, 4) x1, y1 = rpc1.projection(lon, lat, alt)[:2] x2, y2 = rpc2.projection(lon, lat, alt)[:2] m = np.vstack([x1, y1, x2, y2]).T m = np.vstack({tuple(row) for row in m}) # remove duplicates due to no alt range H2 = register_horizontally_shear(m, H1, H2) # compose H2 with a horizontal translation to center disp range around 0 if sift_matches is not None: sift_matches = filter_matches_epipolar_constraint( F, sift_matches, cfg['epipolar_thresh']) if len(sift_matches) < 1: warnings.warn( "Need at least one sift match for the horizontal registration", category=NoHorizontalRegistrationWarning, ) else: H2 = register_horizontally_translation(sift_matches, H1, H2) # compute disparity range if cfg['debug']: out_dir = os.path.dirname(out1) np.savetxt(os.path.join(out_dir, 'sift_matches_disp.txt'), sift_matches, fmt='%9.3f') visualisation.plot_matches( im1, im2, rpc1, rpc2, sift_matches, x, y, w, h, os.path.join(out_dir, 'sift_matches_disp.png')) disp_m, disp_M = disparity_range(rpc1, rpc2, x, y, w, h, H1, H2, sift_matches, A) # recompute hmargin and homographies hmargin = int(np.ceil(max([hmargin, np.fabs(disp_m), np.fabs(disp_M)]))) T = common.matrix_translation(hmargin, vmargin) H1, H2 = np.dot(T, H1), np.dot(T, H2) # compute output images size roi = [[x, y], [x + w, y], [x + w, y + h], [x, y + h]] pts1 = common.points_apply_homography(H1, roi) x0, y0, w0, h0 = common.bounding_box2D(pts1) # check that the first homography maps the ROI in the positive quadrant np.testing.assert_allclose(np.round([x0, y0]), [hmargin, vmargin], atol=.01) # apply homographies and do the crops common.image_apply_homography(out1, im1, H1, w0 + 2 * hmargin, h0 + 2 * vmargin) common.image_apply_homography(out2, im2, H2, w0 + 2 * hmargin, h0 + 2 * vmargin) return H1, H2, disp_m, disp_M
def rectify_pair(im1, im2, rpc1, rpc2, x, y, w, h, out1, out2, A=None, sift_matches=None, method='rpc', hmargin=0, vmargin=0): """ Rectify a ROI in a pair of images. Args: im1, im2: paths to two image files rpc1, rpc2: paths to the two xml files containing RPC data x, y, w, h: four integers defining the rectangular ROI in the first image. (x, y) is the top-left corner, and (w, h) are the dimensions of the rectangle. out1, out2: paths to the output rectified crops A (optional): 3x3 numpy array containing the pointing error correction for im2. This matrix is usually estimated with the pointing_accuracy module. sift_matches (optional): Nx4 numpy array containing a list of sift matches, in the full image coordinates frame method (default: 'rpc'): option to decide wether to use rpc of sift matches for the fundamental matrix estimation. {h,v}margin (optional): horizontal and vertical margins added on the sides of the rectified images Returns: H1, H2: Two 3x3 matrices representing the rectifying homographies that have been applied to the two original (large) images. disp_min, disp_max: horizontal disparity range """ # read RPC data rpc1 = rpc_model.RPCModel(rpc1) rpc2 = rpc_model.RPCModel(rpc2) # compute real or virtual matches if method == 'rpc': # find virtual matches from RPC camera models matches = rpc_utils.matches_from_rpc(rpc1, rpc2, x, y, w, h, cfg['n_gcp_per_axis']) # correct second image coordinates with the pointing correction matrix if A is not None: matches[:, 2:] = common.points_apply_homography( np.linalg.inv(A), matches[:, 2:]) else: matches = sift_matches # compute rectifying homographies H1, H2, F = rectification_homographies(matches, x, y, w, h) if cfg['register_with_shear']: # compose H2 with a horizontal shear to reduce the disparity range a = np.mean(rpc_utils.altitude_range(rpc1, x, y, w, h)) lon, lat, alt = rpc_utils.ground_control_points( rpc1, x, y, w, h, a, a, 4) x1, y1 = rpc1.inverse_estimate(lon, lat, alt)[:2] x2, y2 = rpc2.inverse_estimate(lon, lat, alt)[:2] m = np.vstack([x1, y1, x2, y2]).T m = np.vstack({tuple(row) for row in m}) # remove duplicates due to no alt range H2 = register_horizontally_shear(m, H1, H2) # compose H2 with a horizontal translation to center disp range around 0 if sift_matches is not None: sift_matches = filter_matches_epipolar_constraint( F, sift_matches, cfg['epipolar_thresh']) if len(sift_matches) < 10: print('WARNING: no registration with less than 10 matches') else: H2 = register_horizontally_translation(sift_matches, H1, H2) # compute disparity range if cfg['debug']: out_dir = os.path.dirname(out1) np.savetxt(os.path.join(out_dir, 'sift_matches_disp.txt'), sift_matches, fmt='%9.3f') visualisation.plot_matches( im1, im2, rpc1, rpc2, sift_matches, x, y, w, h, os.path.join(out_dir, 'sift_matches_disp.png')) disp_m, disp_M = disparity_range(rpc1, rpc2, x, y, w, h, H1, H2, sift_matches, A) # recompute hmargin and homographies hmargin = int(np.ceil(max([hmargin, np.fabs(disp_m), np.fabs(disp_M)]))) T = common.matrix_translation(hmargin, vmargin) H1, H2 = np.dot(T, H1), np.dot(T, H2) # compute rectifying homographies for non-epipolar mode (rectify the secondary tile only) if block_matching.rectify_secondary_tile_only(cfg['matching_algorithm']): H1_inv = np.linalg.inv(H1) H1 = np.eye( 3 ) # H1 is replaced by 2-D array with ones on the diagonal and zeros elsewhere H2 = np.dot(H1_inv, H2) T = common.matrix_translation(-x + hmargin, -y + vmargin) H1 = np.dot(T, H1) H2 = np.dot(T, H2) # compute output images size roi = [[x, y], [x + w, y], [x + w, y + h], [x, y + h]] pts1 = common.points_apply_homography(H1, roi) x0, y0, w0, h0 = common.bounding_box2D(pts1) # check that the first homography maps the ROI in the positive quadrant np.testing.assert_allclose(np.round([x0, y0]), [hmargin, vmargin], atol=.01) # apply homographies and do the crops common.image_apply_homography(out1, im1, H1, w0 + 2 * hmargin, h0 + 2 * vmargin) common.image_apply_homography(out2, im2, H2, w0 + 2 * hmargin, h0 + 2 * vmargin) if block_matching.rectify_secondary_tile_only(cfg['matching_algorithm']): pts_in = [[0, 0], [disp_m, 0], [disp_M, 0]] pts_out = common.points_apply_homography(H1_inv, pts_in) disp_m = pts_out[1, :] - pts_out[0, :] disp_M = pts_out[2, :] - pts_out[0, :] return H1, H2, disp_m, disp_M
def disparity_to_ply(tile): """ Compute a point cloud from the disparity map of a pair of image tiles. Args: tile: dictionary containing the information needed to process a tile. """ out_dir = os.path.join(tile['dir']) ply_file = os.path.join(out_dir, 'cloud.ply') plyextrema = os.path.join(out_dir, 'plyextrema.txt') x, y, w, h = tile['coordinates'] rpc1 = cfg['images'][0]['rpc'] rpc2 = cfg['images'][1]['rpc'] if os.path.exists(os.path.join(out_dir, 'stderr.log')): print('triangulation: stderr.log exists') print('pair_1 not processed on tile {} {}'.format(x, y)) return print('triangulating tile {} {}...'.format(x, y)) # This function is only called when there is a single pair (pair_1) H_ref = os.path.join(out_dir, 'pair_1', 'H_ref.txt') H_sec = os.path.join(out_dir, 'pair_1', 'H_sec.txt') pointing = os.path.join(cfg['out_dir'], 'global_pointing_pair_1.txt') disp = os.path.join(out_dir, 'pair_1', 'rectified_disp.tif') extra = os.path.join(out_dir, 'pair_1', 'rectified_disp_confidence.tif') if not os.path.exists(extra): extra = '' mask_rect = os.path.join(out_dir, 'pair_1', 'rectified_mask.png') mask_orig = os.path.join(out_dir, 'cloud_water_image_domain_mask.png') # prepare the image needed to colorize point cloud colors = os.path.join(out_dir, 'rectified_ref.png') if cfg['images'][0]['clr']: hom = np.loadtxt(H_ref) roi = [[x, y], [x + w, y], [x + w, y + h], [x, y + h]] ww, hh = common.bounding_box2D(common.points_apply_homography( hom, roi))[2:] tmp = common.tmpfile('.tif') common.image_apply_homography(tmp, cfg['images'][0]['clr'], hom, ww + 2 * cfg['horizontal_margin'], hh + 2 * cfg['vertical_margin']) common.image_qauto(tmp, colors) else: common.image_qauto( os.path.join(out_dir, 'pair_1', 'rectified_ref.tif'), colors) # compute the point cloud triangulation.disp_map_to_point_cloud(ply_file, disp, mask_rect, rpc1, rpc2, H_ref, H_sec, pointing, colors, extra, utm_zone=cfg['utm_zone'], llbbx=tuple(cfg['ll_bbx']), xybbx=(x, x + w, y, y + h), xymsk=mask_orig) # compute the point cloud extrema (xmin, xmax, xmin, ymax) common.run("plyextrema %s %s" % (ply_file, plyextrema)) if cfg['clean_intermediate']: common.remove(H_ref) common.remove(H_sec) common.remove(disp) common.remove(mask_rect) common.remove(mask_orig) common.remove(colors) common.remove(os.path.join(out_dir, 'pair_1', 'rectified_ref.tif'))