def register_horizontally_shear(matches, H1, H2):
"""
Adjust rectifying homographies with tilt, shear and translation to reduce the disparity range.
Args:
matches: list of pairs of 2D points, stored as a Nx4 numpy array
H1, H2: two homographies, stored as numpy 3x3 matrices
Returns:
H2: corrected homography H2
The matches are provided in the original images coordinate system. By
transforming these coordinates with the provided homographies, we obtain
matches whose disparity is only along the x-axis.
"""
# transform the matches according to the homographies
p1 = common.points_apply_homography(H1, matches[:, :2])
x1 = p1[:, 0]
y1 = p1[:, 1]
p2 = common.points_apply_homography(H2, matches[:, 2:])
x2 = p2[:, 0]
y2 = p2[:, 1]
if cfg['debug']:
print("Residual vertical disparities: max, min, mean. Should be zero")
print(np.max(y2 - y1), np.min(y2 - y1), np.mean(y2 - y1))
# we search the (a, b, c) vector that minimises \sum (x1 - (a*x2+b*y2+c))^2
# it is a least squares minimisation problem
A = np.vstack((x2, y2, y2 * 0 + 1)).T
a, b, c = np.linalg.lstsq(A, x1)[0].flatten()
# correct H2 with the estimated tilt, shear and translation
return np.dot(np.array([[a, b, c], [0, 1, 0], [0, 0, 1]]), H2)
def disparity_range_from_matches(matches, H1, H2, w, h):
"""
Compute the disparity range of a ROI from a list of point matches.
Args:
matches: Nx4 numpy array containing a list of matches, in the full
image coordinates frame, before rectification
w, h: width and height of the rectangular ROI in the first image.
H1, H2: two rectifying homographies, stored as numpy 3x3 matrices
Returns:
disp_min, disp_max: horizontal disparity range
"""
# transform the matches according to the homographies
p1 = common.points_apply_homography(H1, matches[:, :2])
x1 = p1[:, 0]
p2 = common.points_apply_homography(H2, matches[:, 2:])
x2 = p2[:, 0]
# compute the final disparity range
disp_min = np.floor(np.min(x2 - x1))
disp_max = np.ceil(np.max(x2 - x1))
# add a security margin to the disparity range
disp_min -= (disp_max - disp_min) * cfg['disp_range_extra_margin']
disp_max += (disp_max - disp_min) * cfg['disp_range_extra_margin']
return disp_min, disp_max
def alt_to_disp(rpc1, rpc2, x, y, alt, H1, H2, A=None):
"""
Converts an altitude into a disparity.
Args:
rpc1: an instance of the rpcm.RPCModel class for the reference
image
rpc2: an instance of the rpcm.RPCModel class for the secondary
image
x, y: coordinates of the point in the reference image
alt: altitude above the WGS84 ellipsoid (in meters) of the point
H1, H2: rectifying homographies
A (optional): pointing correction matrix
Returns:
the horizontal disparity of the (x, y) point of im1, assuming that the
3-space point associated has altitude alt. The disparity is made
horizontal thanks to the two rectifying homographies H1 and H2.
"""
xx, yy = find_corresponding_point(rpc1, rpc2, x, y, alt)[0:2]
p1 = np.vstack([x, y]).T
p2 = np.vstack([xx, yy]).T
if A is not None:
print("rpc_utils.alt_to_disp: applying pointing error correction")
# correct coordinates of points in im2, according to A
p2 = common.points_apply_homography(np.linalg.inv(A), p2)
p1 = common.points_apply_homography(H1, p1)
p2 = common.points_apply_homography(H2, p2)
# np.testing.assert_allclose(p1[:, 1], p2[:, 1], atol=0.1)
disp = p2[:, 0] - p1[:, 0]
return disp
def test_affine_transformation():
x = np.array([[0, 0], [0, 1], [1, 0], [1, 1]])
# list of transformations to be tested
T = np.eye(3)
I = np.eye(3)
S = np.eye(3)
A = np.eye(3)
translations = []
isometries = []
similarities = []
affinities = []
for i in range(100):
translations.append(T)
isometries.append(I)
similarities.append(S)
affinities.append(A)
T[0:2, 2] = np.random.random(2)
I = rotation_matrix(2 * np.pi * np.random.random_sample())
I[0:2, 2] = np.random.random(2)
S = similarity_matrix(2 * np.pi * np.random.random_sample(),
np.random.random_sample())
S[0:2, 2] = 100 * np.random.random(2)
A[0:2, :] = np.random.random((2, 3))
for B in translations + isometries + similarities + affinities:
xx = common.points_apply_homography(B, x)
E = estimation.affine_transformation(x, xx)
assert_array_almost_equal(E, B)
def register_horizontally_translation(matches, H1, H2, flag='center'):
"""
Adjust rectifying homographies with a translation to modify the disparity range.
Args:
matches: list of pairs of 2D points, stored as a Nx4 numpy array
H1, H2: two homographies, stored as numpy 3x3 matrices
flag: option needed to control how to modify the disparity range:
'center': move the barycenter of disparities of matches to zero
'positive': make all the disparities positive
'negative': make all the disparities negative. Required for
Hirshmuller stereo (java)
Returns:
H2: corrected homography H2
The matches are provided in the original images coordinate system. By
transforming these coordinates with the provided homographies, we obtain
matches whose disparity is only along the x-axis. The second homography H2
is corrected with a horizontal translation to obtain the desired property
on the disparity range.
"""
# transform the matches according to the homographies
p1 = common.points_apply_homography(H1, matches[:, :2])
x1 = p1[:, 0]
y1 = p1[:, 1]
p2 = common.points_apply_homography(H2, matches[:, 2:])
x2 = p2[:, 0]
y2 = p2[:, 1]
# for debug, print the vertical disparities. Should be zero.
if cfg['debug']:
print("Residual vertical disparities: max, min, mean. Should be zero")
print(np.max(y2 - y1), np.min(y2 - y1), np.mean(y2 - y1))
# compute the disparity offset according to selected option
t = 0
if (flag == 'center'):
t = np.mean(x2 - x1)
if (flag == 'positive'):
t = np.min(x2 - x1)
if (flag == 'negative'):
t = np.max(x2 - x1)
# correct H2 with a translation
return np.dot(common.matrix_translation(-t, 0), H2)
def rectification_homographies(matches, x, y, w, h):
"""
Computes rectifying homographies from point matches for a given ROI.
The affine fundamental matrix F is estimated with the gold-standard
algorithm, then two rectifying similarities (rotation, zoom, translation)
are computed directly from F.
Args:
matches: numpy array of shape (n, 4) containing a list of 2D point
correspondences between the two images.
x, y, w, h: four integers defining the rectangular ROI in the first
image. (x, y) is the top-left corner, and (w, h) are the dimensions
of the rectangle.
Returns:
S1, S2, F: three numpy arrays of shape (3, 3) representing the
two rectifying similarities to be applied to the two images and the
corresponding affine fundamental matrix.
"""
# estimate the affine fundamental matrix with the Gold standard algorithm
F = estimation.affine_fundamental_matrix(matches)
# compute rectifying similarities
S1, S2 = estimation.rectifying_similarities_from_affine_fundamental_matrix(
F, cfg['debug'])
if cfg['debug']:
y1 = common.points_apply_homography(S1, matches[:, :2])[:, 1]
y2 = common.points_apply_homography(S2, matches[:, 2:])[:, 1]
err = np.abs(y1 - y2)
print("max, min, mean rectification error on point matches: ", end=' ')
print(np.max(err), np.min(err), np.mean(err))
# pull back top-left corner of the ROI to the origin (plus margin)
pts = common.points_apply_homography(
S1, [[x, y], [x + w, y], [x + w, y + h], [x, y + h]])
x0, y0 = common.bounding_box2D(pts)[:2]
T = common.matrix_translation(-x0, -y0)
return np.dot(T, S1), np.dot(T, S2), F
def rectify_pair(im1,
im2,
rpc1,
rpc2,
x,
y,
w,
h,
out1,
out2,
A=None,
sift_matches=None,
method='rpc',
hmargin=0,
vmargin=0):
"""
Rectify a ROI in a pair of images.
Args:
im1, im2: paths to two GeoTIFF image files
rpc1, rpc2: two instances of the rpcm.RPCModel class
x, y, w, h: four integers defining the rectangular ROI in the first
image. (x, y) is the top-left corner, and (w, h) are the dimensions
of the rectangle.
out1, out2: paths to the output rectified crops
A (optional): 3x3 numpy array containing the pointing error correction
for im2. This matrix is usually estimated with the pointing_accuracy
module.
sift_matches (optional): Nx4 numpy array containing a list of sift
matches, in the full image coordinates frame
method (default: 'rpc'): option to decide wether to use rpc of sift
matches for the fundamental matrix estimation.
{h,v}margin (optional): horizontal and vertical margins added on the
sides of the rectified images
Returns:
H1, H2: Two 3x3 matrices representing the rectifying homographies that
have been applied to the two original (large) images.
disp_min, disp_max: horizontal disparity range
"""
# compute real or virtual matches
if method == 'rpc':
# find virtual matches from RPC camera models
matches = rpc_utils.matches_from_rpc(rpc1, rpc2, x, y, w, h,
cfg['n_gcp_per_axis'])
# correct second image coordinates with the pointing correction matrix
if A is not None:
matches[:, 2:] = common.points_apply_homography(
np.linalg.inv(A), matches[:, 2:])
elif method == 'sift':
matches = sift_matches
else:
raise Exception(
"Unknown value {} for argument 'method'".format(method))
if matches is None or len(matches) < 4:
raise NoRectificationMatchesError(
"No or not enough matches found to rectify image pair")
# compute rectifying homographies
H1, H2, F = rectification_homographies(matches, x, y, w, h)
if cfg['register_with_shear']:
# compose H2 with a horizontal shear to reduce the disparity range
a = np.mean(rpc_utils.altitude_range(rpc1, x, y, w, h))
lon, lat, alt = rpc_utils.ground_control_points(
rpc1, x, y, w, h, a, a, 4)
x1, y1 = rpc1.projection(lon, lat, alt)[:2]
x2, y2 = rpc2.projection(lon, lat, alt)[:2]
m = np.vstack([x1, y1, x2, y2]).T
m = np.vstack({tuple(row)
for row in m}) # remove duplicates due to no alt range
H2 = register_horizontally_shear(m, H1, H2)
# compose H2 with a horizontal translation to center disp range around 0
if sift_matches is not None:
sift_matches = filter_matches_epipolar_constraint(
F, sift_matches, cfg['epipolar_thresh'])
if len(sift_matches) < 1:
warnings.warn(
"Need at least one sift match for the horizontal registration",
category=NoHorizontalRegistrationWarning,
)
else:
H2 = register_horizontally_translation(sift_matches, H1, H2)
# compute disparity range
if cfg['debug']:
out_dir = os.path.dirname(out1)
np.savetxt(os.path.join(out_dir, 'sift_matches_disp.txt'),
sift_matches,
fmt='%9.3f')
visualisation.plot_matches(
im1, im2, rpc1, rpc2, sift_matches, x, y, w, h,
os.path.join(out_dir, 'sift_matches_disp.png'))
disp_m, disp_M = disparity_range(rpc1, rpc2, x, y, w, h, H1, H2,
sift_matches, A)
# recompute hmargin and homographies
hmargin = int(np.ceil(max([hmargin, np.fabs(disp_m), np.fabs(disp_M)])))
T = common.matrix_translation(hmargin, vmargin)
H1, H2 = np.dot(T, H1), np.dot(T, H2)
# compute output images size
roi = [[x, y], [x + w, y], [x + w, y + h], [x, y + h]]
pts1 = common.points_apply_homography(H1, roi)
x0, y0, w0, h0 = common.bounding_box2D(pts1)
# check that the first homography maps the ROI in the positive quadrant
np.testing.assert_allclose(np.round([x0, y0]), [hmargin, vmargin],
atol=.01)
# apply homographies and do the crops
common.image_apply_homography(out1, im1, H1, w0 + 2 * hmargin,
h0 + 2 * vmargin)
common.image_apply_homography(out2, im2, H2, w0 + 2 * hmargin,
h0 + 2 * vmargin)
return H1, H2, disp_m, disp_M
def rectify_pair(im1,
im2,
rpc1,
rpc2,
x,
y,
w,
h,
out1,
out2,
A=None,
sift_matches=None,
method='rpc',
hmargin=0,
vmargin=0):
"""
Rectify a ROI in a pair of images.
Args:
im1, im2: paths to two image files
rpc1, rpc2: paths to the two xml files containing RPC data
x, y, w, h: four integers defining the rectangular ROI in the first
image. (x, y) is the top-left corner, and (w, h) are the dimensions
of the rectangle.
out1, out2: paths to the output rectified crops
A (optional): 3x3 numpy array containing the pointing error correction
for im2. This matrix is usually estimated with the pointing_accuracy
module.
sift_matches (optional): Nx4 numpy array containing a list of sift
matches, in the full image coordinates frame
method (default: 'rpc'): option to decide wether to use rpc of sift
matches for the fundamental matrix estimation.
{h,v}margin (optional): horizontal and vertical margins added on the
sides of the rectified images
Returns:
H1, H2: Two 3x3 matrices representing the rectifying homographies that
have been applied to the two original (large) images.
disp_min, disp_max: horizontal disparity range
"""
# read RPC data
rpc1 = rpc_model.RPCModel(rpc1)
rpc2 = rpc_model.RPCModel(rpc2)
# compute real or virtual matches
if method == 'rpc':
# find virtual matches from RPC camera models
matches = rpc_utils.matches_from_rpc(rpc1, rpc2, x, y, w, h,
cfg['n_gcp_per_axis'])
# correct second image coordinates with the pointing correction matrix
if A is not None:
matches[:, 2:] = common.points_apply_homography(
np.linalg.inv(A), matches[:, 2:])
else:
matches = sift_matches
# compute rectifying homographies
H1, H2, F = rectification_homographies(matches, x, y, w, h)
if cfg['register_with_shear']:
# compose H2 with a horizontal shear to reduce the disparity range
a = np.mean(rpc_utils.altitude_range(rpc1, x, y, w, h))
lon, lat, alt = rpc_utils.ground_control_points(
rpc1, x, y, w, h, a, a, 4)
x1, y1 = rpc1.inverse_estimate(lon, lat, alt)[:2]
x2, y2 = rpc2.inverse_estimate(lon, lat, alt)[:2]
m = np.vstack([x1, y1, x2, y2]).T
m = np.vstack({tuple(row)
for row in m}) # remove duplicates due to no alt range
H2 = register_horizontally_shear(m, H1, H2)
# compose H2 with a horizontal translation to center disp range around 0
if sift_matches is not None:
sift_matches = filter_matches_epipolar_constraint(
F, sift_matches, cfg['epipolar_thresh'])
if len(sift_matches) < 10:
print('WARNING: no registration with less than 10 matches')
else:
H2 = register_horizontally_translation(sift_matches, H1, H2)
# compute disparity range
if cfg['debug']:
out_dir = os.path.dirname(out1)
np.savetxt(os.path.join(out_dir, 'sift_matches_disp.txt'),
sift_matches,
fmt='%9.3f')
visualisation.plot_matches(
im1, im2, rpc1, rpc2, sift_matches, x, y, w, h,
os.path.join(out_dir, 'sift_matches_disp.png'))
disp_m, disp_M = disparity_range(rpc1, rpc2, x, y, w, h, H1, H2,
sift_matches, A)
# recompute hmargin and homographies
hmargin = int(np.ceil(max([hmargin, np.fabs(disp_m), np.fabs(disp_M)])))
T = common.matrix_translation(hmargin, vmargin)
H1, H2 = np.dot(T, H1), np.dot(T, H2)
# compute rectifying homographies for non-epipolar mode (rectify the secondary tile only)
if block_matching.rectify_secondary_tile_only(cfg['matching_algorithm']):
H1_inv = np.linalg.inv(H1)
H1 = np.eye(
3
) # H1 is replaced by 2-D array with ones on the diagonal and zeros elsewhere
H2 = np.dot(H1_inv, H2)
T = common.matrix_translation(-x + hmargin, -y + vmargin)
H1 = np.dot(T, H1)
H2 = np.dot(T, H2)
# compute output images size
roi = [[x, y], [x + w, y], [x + w, y + h], [x, y + h]]
pts1 = common.points_apply_homography(H1, roi)
x0, y0, w0, h0 = common.bounding_box2D(pts1)
# check that the first homography maps the ROI in the positive quadrant
np.testing.assert_allclose(np.round([x0, y0]), [hmargin, vmargin],
atol=.01)
# apply homographies and do the crops
common.image_apply_homography(out1, im1, H1, w0 + 2 * hmargin,
h0 + 2 * vmargin)
common.image_apply_homography(out2, im2, H2, w0 + 2 * hmargin,
h0 + 2 * vmargin)
if block_matching.rectify_secondary_tile_only(cfg['matching_algorithm']):
pts_in = [[0, 0], [disp_m, 0], [disp_M, 0]]
pts_out = common.points_apply_homography(H1_inv, pts_in)
disp_m = pts_out[1, :] - pts_out[0, :]
disp_M = pts_out[2, :] - pts_out[0, :]
return H1, H2, disp_m, disp_M
def disparity_to_ply(tile):
"""
Compute a point cloud from the disparity map of a pair of image tiles.
Args:
tile: dictionary containing the information needed to process a tile.
"""
out_dir = os.path.join(tile['dir'])
ply_file = os.path.join(out_dir, 'cloud.ply')
plyextrema = os.path.join(out_dir, 'plyextrema.txt')
x, y, w, h = tile['coordinates']
rpc1 = cfg['images'][0]['rpc']
rpc2 = cfg['images'][1]['rpc']
if os.path.exists(os.path.join(out_dir, 'stderr.log')):
print('triangulation: stderr.log exists')
print('pair_1 not processed on tile {} {}'.format(x, y))
return
print('triangulating tile {} {}...'.format(x, y))
# This function is only called when there is a single pair (pair_1)
H_ref = os.path.join(out_dir, 'pair_1', 'H_ref.txt')
H_sec = os.path.join(out_dir, 'pair_1', 'H_sec.txt')
pointing = os.path.join(cfg['out_dir'], 'global_pointing_pair_1.txt')
disp = os.path.join(out_dir, 'pair_1', 'rectified_disp.tif')
extra = os.path.join(out_dir, 'pair_1', 'rectified_disp_confidence.tif')
if not os.path.exists(extra):
extra = ''
mask_rect = os.path.join(out_dir, 'pair_1', 'rectified_mask.png')
mask_orig = os.path.join(out_dir, 'cloud_water_image_domain_mask.png')
# prepare the image needed to colorize point cloud
colors = os.path.join(out_dir, 'rectified_ref.png')
if cfg['images'][0]['clr']:
hom = np.loadtxt(H_ref)
roi = [[x, y], [x + w, y], [x + w, y + h], [x, y + h]]
ww, hh = common.bounding_box2D(common.points_apply_homography(
hom, roi))[2:]
tmp = common.tmpfile('.tif')
common.image_apply_homography(tmp, cfg['images'][0]['clr'], hom,
ww + 2 * cfg['horizontal_margin'],
hh + 2 * cfg['vertical_margin'])
common.image_qauto(tmp, colors)
else:
common.image_qauto(
os.path.join(out_dir, 'pair_1', 'rectified_ref.tif'), colors)
# compute the point cloud
triangulation.disp_map_to_point_cloud(ply_file,
disp,
mask_rect,
rpc1,
rpc2,
H_ref,
H_sec,
pointing,
colors,
extra,
utm_zone=cfg['utm_zone'],
llbbx=tuple(cfg['ll_bbx']),
xybbx=(x, x + w, y, y + h),
xymsk=mask_orig)
# compute the point cloud extrema (xmin, xmax, xmin, ymax)
common.run("plyextrema %s %s" % (ply_file, plyextrema))
if cfg['clean_intermediate']:
common.remove(H_ref)
common.remove(H_sec)
common.remove(disp)
common.remove(mask_rect)
common.remove(mask_orig)
common.remove(colors)
common.remove(os.path.join(out_dir, 'pair_1', 'rectified_ref.tif'))
