def plot_matches(im1, im2, p1, p2):
h1, w1, c = im1.shape
h2, w2, c = im2.shape
image = np.zeros((max(h1, h2), w1 + w2, 3), dtype=im1.dtype)
image[0:h1, 0:w1, :] = im1
image[0:h2, w1:(w1 + w2), :] = im2
p1 = features.denormalized_image_coordinates(p1, w1, h1)
p2 = features.denormalized_image_coordinates(p2, w2, h2)
pl.imshow(image)
for a, b in zip(p1, p2):
pl.plot([a[0], b[0] + w1], [a[1], b[1]], "c")
pl.plot(p1[:, 0], p1[:, 1], "ob")
pl.plot(p2[:, 0] + w1, p2[:, 1], "ob")
def render_perspective_view_of_a_panorama(image, panoshot, perspectiveshot):
"""Render a perspective view of a panorama."""
# Get destination pixel coordinates
dst_shape = (perspectiveshot.camera.height, perspectiveshot.camera.width)
dst_y, dst_x = np.indices(dst_shape).astype(np.float32)
dst_pixels_denormalized = np.column_stack([dst_x.ravel(), dst_y.ravel()])
dst_pixels = features.normalized_image_coordinates(
dst_pixels_denormalized, perspectiveshot.camera.width,
perspectiveshot.camera.height)
# Convert to bearing
dst_bearings = perspectiveshot.camera.pixel_bearings(dst_pixels)
# Rotate to panorama reference frame
rotation = np.dot(panoshot.pose.get_rotation_matrix(),
perspectiveshot.pose.get_rotation_matrix().T)
rotated_bearings = np.dot(dst_bearings, rotation.T)
# Project to panorama pixels
src_x, src_y = panoshot.camera.project(
(rotated_bearings[:, 0], rotated_bearings[:, 1], rotated_bearings[:,
2]))
src_pixels = np.column_stack([src_x.ravel(), src_y.ravel()])
src_pixels_denormalized = features.denormalized_image_coordinates(
src_pixels, image.shape[1], image.shape[0])
# Sample color
colors = cv2.remap(image, src_pixels_denormalized[:, 0].astype(np.float32),
src_pixels_denormalized[:, 1].astype(np.float32),
cv2.INTER_LINEAR)
colors.shape = dst_shape + (-1, )
return colors
def render_perspective_view_of_a_panorama(image, panoshot, perspectiveshot,
interpolation=cv2.INTER_LINEAR,
borderMode=cv2.BORDER_WRAP):
"""Render a perspective view of a panorama."""
# Get destination pixel coordinates
dst_shape = (perspectiveshot.camera.height, perspectiveshot.camera.width)
dst_y, dst_x = np.indices(dst_shape).astype(np.float32)
dst_pixels_denormalized = np.column_stack([dst_x.ravel(), dst_y.ravel()])
dst_pixels = features.normalized_image_coordinates(
dst_pixels_denormalized,
perspectiveshot.camera.width,
perspectiveshot.camera.height)
# Convert to bearing
dst_bearings = perspectiveshot.camera.pixel_bearing_many(dst_pixels)
# Rotate to panorama reference frame
rotation = np.dot(panoshot.pose.get_rotation_matrix(),
perspectiveshot.pose.get_rotation_matrix().T)
rotated_bearings = np.dot(dst_bearings, rotation.T)
# Project to panorama pixels
src_pixels = panoshot.camera.project_many(rotated_bearings)
src_pixels_denormalized = features.denormalized_image_coordinates(
src_pixels, image.shape[1], image.shape[0])
src_pixels_denormalized.shape = dst_shape + (2,)
# Sample color
x = src_pixels_denormalized[..., 0].astype(np.float32)
y = src_pixels_denormalized[..., 1].astype(np.float32)
colors = cv2.remap(image, x, y, interpolation, borderMode=borderMode)
return colors
def show_epipolar_lines(self, main_image_idx):
if len(self.views) > 2:
raise NotImplementedError("Not implemented yet for >2 views")
img1 = self.views[0].current_image
img2 = self.views[1].current_image
img1_size = self.database.get_image_size(img1)
img2_size = self.database.get_image_size(img2)
matched_points = self.database.get_visible_points_coords(
self.views[main_image_idx].current_image)
matched_points_coords = convert_tuple_cords_to_list(matched_points)
matched_points_coords = features.normalized_image_coordinates(
matched_points_coords, img1_size[1], img1_size[0])
color_idx = 0
for point_idx, point in enumerate(matched_points_coords):
image_pair = [img1, img2]
line = calc_epipol_line(point, image_pair,
self.database.get_path(), main_image_idx)
denormalized_lines = features.denormalized_image_coordinates(
line, img2_size[1], img2_size[0])
for line_segment in denormalized_lines:
circle = mpatches.Circle(
(line_segment[0], line_segment[1]),
3,
color=distinct_colors[divmod(
hash(list(matched_points.keys())[point_idx]), 19)[1]])
self.views[main_image_idx].plt_artists.append(circle)
self.views[not main_image_idx].subplot.add_artist(circle)
color_idx = color_idx + 1
self.views[not main_image_idx].figure.canvas.draw_idle()
def paint_reconstruction(data, graph, reconstruction):
to_paint = defaultdict(list)
to_paint_track = defaultdict(list)
for track in reconstruction['points']:
for shot in graph[track]:
to_paint[shot].append(graph[track][shot]['feature'])
to_paint_track[shot].append(track)
track_colors = {track: np.zeros(3) for track in reconstruction['points']}
track_sum = {track: 0 for track in reconstruction['points']}
for shot in to_paint:
points = np.array(to_paint[shot])
tracks = to_paint_track[shot]
im = data.image_as_array(shot)
pixels = features.denormalized_image_coordinates(
points, im.shape[1], im.shape[0]).astype(int)
colors = im[pixels[:, 1], pixels[:, 0]]
for track, color in zip(tracks, colors):
track_colors[track] += color
track_sum[track] += 1
for track in reconstruction['points']:
c = track_colors[track] / track_sum[track]
reconstruction['points'][track]['color'] = list(c)
def init_points(self, points):
for point in points:
point_id, observations = point['id'], point['observations']
for observation in observations:
h, w = self.get_image_size(observation["shot_id"])
observation["projection"] = features.denormalized_image_coordinates(
np.array([observation["projection"]]), w, h)[0]
self.points[point_id] = observations
def project_pointcloud_save_depth(udata, urec, points, shot_id, max_sz):
# Project points to the undistorted image
shot = urec.shots[shot_id]
w, h = shot.camera.width, shot.camera.height
large = max(w, h)
if large > max_sz:
ar = w / h
if w > h:
w = max_sz
h = int(w / ar)
else:
h = max_sz
w = int(ar * h)
points_2d = shot.project_many(points)
pixel_coords = features.denormalized_image_coordinates(points_2d, w,
h).astype(int)
# Filter out points that fall out of the image
# <<< aren't we supposed to have points that are visible from this image only??!?!
mask = np.ones(pixel_coords.shape[0], dtype=bool)
mask[pixel_coords[:, 0] < 0] = 0
mask[pixel_coords[:, 1] < 0] = 0
mask[pixel_coords[:, 0] >= w] = 0
mask[pixel_coords[:, 1] >= h] = 0
pixel_coords = pixel_coords[mask]
# Compute the depth
distances = np.linalg.norm(points - shot.pose.get_origin(), axis=1)
viewing_angles = np.arctan2(np.linalg.norm(points_2d, axis=1),
shot.camera.focal)
depths = distances * np.cos(viewing_angles)
depths[depths > udata.config["depthmap_max_depth"]] = 0
# Create depth image
depth_image = np.zeros([h, w])
depth_image[pixel_coords[:, 1], pixel_coords[:, 0]] = depths[mask]
# Save numpy
filepath = Path(udata.depthmap_file(shot_id, "clean.npz"))
filepath.parent.mkdir(exist_ok=True, parents=True)
np.savez_compressed(filepath,
depth=depth_image,
plane=np.zeros(1),
score=np.zeros(1))
# Save jpg for visualization
import matplotlib.pyplot as plt
fig = plt.figure()
rgb, sm = depth_colormap(depth_image)
plt.imshow(rgb)
small_colorbar(plt.gca(), mappable=sm)
filepath = Path(
udata.data_path) / "plot_depthmaps" / "{}.png".format(shot_id)
filepath.parent.mkdir(exist_ok=True, parents=True)
plt.savefig(filepath, dpi=300)
plt.close(fig)
def export_features(data, db, images_map):
features_map = {}
for image in data.images():
width = data.load_exif(image)['width']
height = data.load_exif(image)['height']
feat, _, _ = data.load_features(image)
feat = features.denormalized_image_coordinates(feat, width, height)
features_map[image] = feat
db.add_keypoints(images_map[image], feat)
return features_map
def gcp_to_pixel_coordinates(self, x: float,
y: float) -> Tuple[float, float]:
"""
Transforms from normalized coordinates (in the whole geotiff) to
pixels (in the viewing window)
"""
h, w = self.image_manager.get_image_size(self.current_image)
px = features.denormalized_image_coordinates(np.array([[x, y]]), w,
h)[0]
x = px[0] - self.image_window.col_off
y = px[1] - self.image_window.row_off
return [x, y]
def load_gcp_reprojections(self, filename):
with open(filename, 'r') as f:
d = json.load(f)
for gcp_id in d:
for shot_id in d[gcp_id]:
h, w = self.get_image_size(shot_id)
reproj = d[gcp_id][shot_id]['reprojection']
reproj = features.denormalized_image_coordinates(np.array([reproj]), w, h)[0]
d[gcp_id][shot_id]['reprojection'] = reproj
self.gcp_reprojections = d
def gcp_to_pixel_coordinates(self, x: float,
y: float) -> Tuple[float, float]:
"""
Transforms from normalized coordinates to pixels
The view displays images at a reduced resolution for speed. We use the image
manager to obtain the reduced coordinates to use for de-normalization.
"""
h, w = self.image_manager.get_image_size(self.current_image)
px = features.denormalized_image_coordinates(np.array([[x, y]]), w,
h)[0]
return self.rotate_point(px[0], px[1], h, w, reverse=False)
def save_features_json(self, filepath, points, image):
img = cv2.imread(self._image_file(image))
print('image:', image)
print('img:', img)
print('image.shape', img.shape)
h, w, d = img.shape
denormed_points = features.denormalized_image_coordinates(points, w, h)
_out = {}
for i, item in enumerate(denormed_points):
_out[str(i)] = list(map(str, list(item)))
print(_out)
with open(filepath + ".json", "w") as f:
json.dump(_out, f, indent=4)
def export_features(data, db, images_map):
features_map = {}
for image in data.images():
width = data.load_exif(image)["width"]
height = data.load_exif(image)["height"]
features_data = data.load_features(image)
if not features_data:
continue
feat = features.denormalized_image_coordinates(features_data.points,
width, height)
features_map[image] = feat
db.add_keypoints(images_map[image], feat)
return features_map
def check_a_point(point, mask):
# assert that the point is one dimensional
points = np.array(point)
points = points[np.newaxis, 0:2]
points = features.denormalized_image_coordinates(points, mask.shape[1],
mask.shape[0])
p = points[0, :]
if mask[int(p[1]), int(p[0])] == 0:
# then not include this point
return 0
else:
# include this point
return 1
def gcp_to_pixel_coordinates(self, x: float,
y: float) -> Tuple[float, float]:
"""
Transforms from normalized coordinates (in the whole geotiff) to
pixels (in the viewing window)
"""
h, w = self.image_manager.get_image_size(self.current_image)
px = features.denormalized_image_coordinates(np.array([[x, y]]), w,
h)[0]
# pyre-fixme[16]: `OrthoPhotoView` has no attribute `image_window`.
x = px[0] - self.image_window.col_off
y = px[1] - self.image_window.row_off
# pyre-fixme[7]: Expected `Tuple[float, float]` but got `List[typing.Any]`.
return [x, y]
def filter_by_seg(self, image, points, func, relative_seg_path):
# return a list of true or false indicating whether the point survive after mask
seg = self.seg_as_array(image, relative_seg_path)
points = points[:, 0:2]
points = features.denormalized_image_coordinates(
points, seg.shape[1], seg.shape[0])
ans = []
for p in points:
if func(seg[int(p[1]), int(p[0])]):
ans.append(True)
else:
ans.append(False)
return np.array(ans)
def show_epipolar_lines(self, main_image_idx):
img1_size = self.database.get_image_size(self.curr_images[0])
img2_size = self.database.get_image_size(self.curr_images[1])
matched_points = self.database.get_visible_points_coords(self.curr_images[main_image_idx])
matched_points_coords = convert_tuple_cords_to_list(matched_points)
matched_points_coords = features.normalized_image_coordinates(matched_points_coords, img1_size[1], img1_size[0])
color_idx = 0
for point_idx, point in enumerate(matched_points_coords):
line = calc_epipol_line(point, self.curr_images, self.database.get_path(), main_image_idx)
denormalized_lines = features.denormalized_image_coordinates(line, img2_size[1], img2_size[0])
for line_segment in denormalized_lines:
circle = mpatches.Circle((line_segment[0], line_segment[1]), 3,
color=distinct_colors[divmod(hash(list(matched_points.keys())[point_idx]), 19)[1]])
self.plt_artists[main_image_idx].append(circle)
self.subplots[not main_image_idx].add_artist(circle)
color_idx = color_idx + 1
self.figures[not main_image_idx].canvas.draw_idle()
def display_features(file_path, filename, config, n_fts):
"""
Inputs:
file_path: where the images, features are stored
config: SfM configuration info
n_fts: display every n features
Outputs:
will display the image along with every n_fts on it
"""
fts = features.load_features(file_path + "features/" + filename, config)
points = fts[0]
img_name = filename[:-13]
img = cv2.imread(file_path + "images/" + img_name)
plt.imshow(img)
denorm_pts = features.denormalized_image_coordinates(points, 1920, 1080)
for pt_i, (x, y) in enumerate(denorm_pts):
if (pt_i % n_fts) == 0:
plt.scatter(x, y)
plt.show()
def render_perspective_view_of_a_panorama(image, panoshot, perspectiveshot,
interpolation=cv2.INTER_LINEAR):
"""Render a perspective view of a panorama."""
# Get destination pixel coordinates
dst_shape = (perspectiveshot.camera.height, perspectiveshot.camera.width)
dst_y, dst_x = np.indices(dst_shape).astype(np.float32)
dst_pixels_denormalized = np.column_stack([dst_x.ravel(), dst_y.ravel()])
dst_pixels = features.normalized_image_coordinates(
dst_pixels_denormalized,
perspectiveshot.camera.width,
perspectiveshot.camera.height)
# Convert to bearing
dst_bearings = perspectiveshot.camera.pixel_bearing_many(dst_pixels)
# Rotate to panorama reference frame
rotation = np.dot(panoshot.pose.get_rotation_matrix(),
perspectiveshot.pose.get_rotation_matrix().T)
rotated_bearings = np.dot(dst_bearings, rotation.T)
# Project to panorama pixels
src_x, src_y = panoshot.camera.project((rotated_bearings[:, 0],
rotated_bearings[:, 1],
rotated_bearings[:, 2]))
src_pixels = np.column_stack([src_x.ravel(), src_y.ravel()])
src_pixels_denormalized = features.denormalized_image_coordinates(
src_pixels, image.shape[1], image.shape[0])
src_pixels_denormalized.shape = dst_shape + (2,)
# Sample color
x = src_pixels_denormalized[..., 0].astype(np.float32)
y = src_pixels_denormalized[..., 1].astype(np.float32)
colors = cv2.remap(image, x, y, interpolation, borderMode=cv2.BORDER_WRAP)
return colors
def getPointColorFromReconstruction(reconstruction, image_dir, point):
label_list = []
for shot_id in reconstruction.shots:
shot = reconstruction.shots[shot_id]
norm_projection = shot.project(point)
image_path = os.path.join(image_dir,
os.path.splitext(shot.id)[0] + ".png")
# print("Projecting onto ", image_path)
image = cv2.imread(image_path)
pt_im = features.denormalized_image_coordinates(
np.array(norm_projection).reshape(-1, 2), image.shape[1],
image.shape[0])[0]
# print(pt_im)
row = int(pt_im[1])
col = int(pt_im[0])
if ((row in range(0, image.shape[0])
and (col in range(0, image.shape[0])))):
rgb = image[row, col]
# print(rgb)
if (np.all(rgb == np.array([98, 98, 98]))):
rgb = 1
elif (np.all(rgb == np.array([100, 100, 100]))):
rgb = 1
else:
rgb = 2
label_list.append(rgb)
if (len(label_list) == 0):
return None
occurence_count = Counter(label_list)
res = occurence_count.most_common(1)[0][0]
if (res == 1):
res = [0, 255, 255]
else:
res = [0, 0, 255]
return res
def detect(args):
image, tags, data = args
logger.info('Extracting {} features for image {}'.format(
data.feature_type().upper(), image))
DEBUG = 0
# check if features already exist
if not data.feature_index_exists(image):
mask = data.mask_as_array(image)
if mask is not None:
logger.info('Found mask to apply for image {}'.format(image))
preemptive_max = data.config.get('preemptive_max', 200)
p_unsorted, f_unsorted, c_unsorted = features.extract_features(
data.image_as_array(image), data.config, mask)
if len(p_unsorted) == 0:
return
#===== prune features in tags =====#
if data.config.get('prune_features_on_tags', False):
# setup
img = cv2.imread(os.path.join(data.data_path, 'images', image))
[height, width, _] = img.shape
p_denorm = features.denormalized_image_coordinates(
p_unsorted, width, height)
# expand tag contour with grid points beyond unit square
expn = 2.0
gridpts = np.array(
[[-expn, expn], [expn, expn], [expn, -expn], [-expn, -expn]],
dtype='float32')
# find features to prune
rm_list = []
for tag in tags:
# contour from tag region
contours = np.array(tag.corners)
if DEBUG > 0:
for i in range(0, 3):
cv2.line(img, (tag.corners[i, 0].astype('int'),
tag.corners[i, 1].astype('int')),
(tag.corners[i + 1, 0].astype('int'),
tag.corners[i + 1, 1].astype('int')),
[0, 255, 0], 12)
cv2.line(img, (tag.corners[3, 0].astype('int'),
tag.corners[3, 1].astype('int')),
(tag.corners[0, 0].astype('int'),
tag.corners[0, 1].astype('int')), [0, 255, 0],
12)
# scale contour outward
H = np.array(tag.homography, dtype='float32')
contours_expanded = cv2.perspectiveTransform(
np.array([gridpts]), H)
# for each point
for pidx in range(0, len(p_unsorted)):
# point
pt = p_denorm[pidx, 0:2]
# point in contour
inout = cv2.pointPolygonTest(
contours_expanded.astype('int'), (pt[0], pt[1]), False)
# check result
if inout >= 0:
rm_list.append(pidx)
# prune features
p_unsorted = np.delete(p_unsorted, np.array(rm_list), axis=0)
f_unsorted = np.delete(f_unsorted, np.array(rm_list), axis=0)
c_unsorted = np.delete(c_unsorted, np.array(rm_list), axis=0)
# debug
if DEBUG > 0:
p_denorm = np.delete(p_denorm, np.array(rm_list), axis=0)
for pidx in range(0, len(p_denorm)):
pt = p_denorm[pidx, 0:2]
cv2.circle(img, (pt[0].astype('int'), pt[1].astype('int')),
5, [0, 0, 255], -1)
cv2.namedWindow('ShowImage', cv2.WINDOW_NORMAL)
height, width, channels = img.shape
showw = max(752, width / 4)
showh = max(480, height / 4)
cv2.resizeWindow('ShowImage', showw, showh)
cv2.imshow('ShowImage', img)
cv2.waitKey(0)
#===== prune features in tags =====#
# sort for preemptive
size = p_unsorted[:, 2]
order = np.argsort(size)
p_sorted = p_unsorted[order, :]
f_sorted = f_unsorted[order, :]
c_sorted = c_unsorted[order, :]
p_pre = p_sorted[-preemptive_max:]
f_pre = f_sorted[-preemptive_max:]
# save
data.save_features(image, p_sorted, f_sorted, c_sorted)
data.save_preemptive_features(image, p_pre, f_pre)
if data.config.get('matcher_type', 'FLANN') == 'FLANN':
index = features.build_flann_index(f_sorted, data.config)
data.save_feature_index(image, index)
#===== tag features =====#
if data.config.get('use_apriltags', False) or data.config.get(
'use_arucotags', False) or data.config.get('use_chromatags',
False):
# setup
try:
tags_all = data.load_tag_detection()
except:
return
tags = tags_all[image]
pt = []
ft = []
ct = []
it = []
imexif = data.load_exif(image)
# for each tag in image
for tag in tags:
# normalize corners
img = cv2.imread(os.path.join(data.data_path, 'images', image))
[height, width, _] = img.shape
#print 'width = ',str(imexif['width'])
#print 'height= ',str(imexif['height'])
#print 'width2= ',str(width)
#print 'heigh2= ',str(height)
norm_tag_corners = features.normalized_image_coordinates(
tag.corners, width,
height) #imexif['width'], imexif['height'])
# for each corner of tag
for r in range(0, 4):
# tag corners
pt.append(norm_tag_corners[r, :])
# tag id
ft.append(tag.id)
# colors
ct.append(tag.colors[r, :])
# corner id (0,1,2,3)
it.append(r)
# if tag features found
if pt:
pt = np.array(pt)
ft = np.array(ft)
ct = np.array(ct)
it = np.array(it)
data.save_tag_features(image, pt, ft, it, ct)
def pix_coords(x, image):
return features.denormalized_image_coordinates(
np.array([[x[0], x[1]]]), image.shape[1], image.shape[0])[0]
def gcp_errors(data: DataSetBase,
reconstructions: List[types.Reconstruction]) -> Dict[str, Any]:
all_errors = []
reference = data.load_reference()
gcps = data.load_ground_control_points()
if not gcps:
return {}
all_errors = []
gcp_stats = []
for gcp in gcps:
if not gcp.lla:
continue
triangulated = None
for rec in reconstructions:
triangulated = multiview.triangulate_gcp(gcp, rec.shots, 1.0, 0.1)
if triangulated is None:
continue
else:
break
if triangulated is None:
continue
gcp_enu = reference.to_topocentric(*gcp.lla_vec)
e = triangulated - gcp_enu
all_errors.append(e)
# Begin computation of GCP stats
observations = []
for i, obs in enumerate(gcp.observations):
if not obs.shot_id in rec.shots:
continue
shot = rec.shots[obs.shot_id]
reprojected = shot.project(gcp_enu)
annotated = obs.projection
r_pixel = features.denormalized_image_coordinates(
np.array([[reprojected[0], reprojected[1]]]),
shot.camera.width, shot.camera.height)[0]
r_pixel[0] /= shot.camera.width
r_pixel[1] /= shot.camera.height
a_pixel = features.denormalized_image_coordinates(
np.array([[annotated[0], annotated[1]]]), shot.camera.width,
shot.camera.height)[0]
a_pixel[0] /= shot.camera.width
a_pixel[1] /= shot.camera.height
observations.append({
'shot_id': obs.shot_id,
'annotated': list(a_pixel),
'reprojected': list(r_pixel)
})
gcp_stats.append({
'id': gcp.id,
'coordinates': list(gcp_enu),
'observations': observations,
'error': list(e)
})
# End computation of GCP stats
with open(
os.path.join(data.data_path, "stats",
"ground_control_points.json"), 'w') as f:
f.write(json.dumps(gcp_stats, indent=4))
return _gps_gcp_errors_stats(np.array(all_errors))
default=3,
type=int,
help='number of points to generate')
args = parser.parse_args()
data = dataset.DataSet(args.dataset)
reference = data.load_reference_lla()
reconstruction = data.load_reconstruction()[0]
print 'WGS84'
for i in range(args.num_points):
point = np.random.choice(reconstruction.points.values())
for shot in reconstruction.shots.values():
pixel = shot.project(point.coordinates)
if np.fabs(pixel).max() < 0.5:
lla = geo.lla_from_topocentric(point.coordinates[0],
point.coordinates[1],
point.coordinates[2],
reference['latitude'],
reference['longitude'],
reference['altitude'])
x, y = features.denormalized_image_coordinates(
pixel.reshape(1, 2), shot.camera.width,
shot.camera.height)[0]
print "{} {} {} {} {} {}".format(lla[0], lla[1], lla[2], x, y,
shot.id)
def create_full_mosaic(args):
log.setup()
shot, data = args
logger.info('Creating full mosaic for image {}'.format( shot.id ) )
config = data.config
start = timer()
projection_type = shot.camera.projection_type
r_map_x = None
r_map_y = None
dst_mask_x = None
dst_mask_y = None
if projection_type in ['perspective', 'brown', 'fisheye']:
img = data.load_image( shot.id )
camera = types.SphericalCamera()
camera.id = "Spherical Projection Camera"
# Determine the correct mosaic size from the focal length of the camera
# Limit this to a maximum of a 16K image which is the highest resolution
# currently supported by PVR.
K_pix = shot.camera.get_K_in_pixel_coordinates()
camera.height = int( np.clip( math.pi*K_pix[0,0], 0, 8192 ) )
camera.width = int( np.clip( 2*math.pi*K_pix[0,0], 0, 16384 ) )
shot_cam = shot.camera
# Project shot's pixels to the spherical mosaic image
src_shape = ( shot_cam.height, shot_cam.width )
src_y, src_x = np.indices( src_shape ).astype( np.float32 )
src_pixels_denormalized = np.column_stack( [ src_x.ravel(), src_y.ravel() ] )
src_pixels = features.normalized_image_coordinates( src_pixels_denormalized, shot_cam.width, shot_cam.height )
# Convert to bearings
src_bearings = shot_cam.pixel_bearing_many( src_pixels )
# Project to spherical mosaic pixels
dst_x, dst_y = camera.project( ( src_bearings[:, 0],
src_bearings[:, 1],
src_bearings[:, 2] ) )
dst_pixels = np.column_stack( [ dst_x.ravel(), dst_y.ravel() ] )
interp_mode = data.config.get( 'full_mosaic_proj_interpolation', 'linear' )
if interp_mode == 'linear':
# Snap to pixel centers to generate a projection index mask. This will be slower then finding
# the ROI using the projected border but it's far easier and covers wrap around and the poles with
# minimal effort. It will also probably be more efficient when wrap around or crossing the poles does occur.
dst_pixels_denormalized_int = features.denormalized_image_coordinates( dst_pixels, camera.width, camera.height ).astype( np.int32 )
dst_pixels_snap = features.normalized_image_coordinates( dst_pixels_denormalized_int.astype( np.float32 ), camera.width, camera.height )
dst_bearings_re = camera.pixel_bearing_many( dst_pixels_snap )
# Project mosaic pixel center bearings back into the source image
src_re_x, src_re_y = shot_cam.project( ( dst_bearings_re[:, 0],
dst_bearings_re[:, 1],
dst_bearings_re[:, 2] ) )
src_re_pixels = np.column_stack( [ src_re_x.ravel(), src_re_y.ravel() ] )
src_re_denormalized = features.denormalized_image_coordinates( src_re_pixels, shot_cam.width, shot_cam.height )
mosaic_img = initialize_mosaic_image( camera.width, camera.height, img )
# Reshape arrays for cv.remap efficiency reasons and due to the SHRT_MAX limit of array size.
# Another option is to process in chunks of linear array of shize SHRT_MAX. However, this
# approach was probably 4x slower.
x = src_re_denormalized[:, 0].reshape( src_x.shape ).astype(np.float32)
y = src_re_denormalized[:, 1].reshape( src_y.shape ).astype(np.float32)
r_map_x = x
r_map_y = y
# Sample source imagery colors
colors = cv2.remap( img, x, y, cv2.INTER_LINEAR , borderMode=cv2.BORDER_CONSTANT )
dst_mask_y = dst_pixels_denormalized_int[:, 1].reshape( src_y.shape )
dst_mask_x = dst_pixels_denormalized_int[:, 0].reshape( src_x.shape )
mosaic_img[ dst_mask_y, dst_mask_x ] = colors
blend_projection_border( mosaic_img, dst_mask_y, dst_mask_x )
# Initialize blurring and alpha mask kernels
# half_chunk_size = 75
# border = 41
# half_size = half_chunk_size + border
# kernel_1d = cv2.getGaussianKernel( 2*half_chunk_size+1, 1.5*(0.3*((2*half_chunk_size+1-1)*0.5 - 1) + 0.8) , cv2.CV_32F )
# kernel_1d/=kernel_1d[ half_chunk_size ]
# half_kernel_1d = kernel_1d[ half_chunk_size : 2*half_chunk_size ]
# alpha = np.zeros( ( 2*half_chunk_size, 2*half_chunk_size, 3 ), dtype = np.float32 ) #np.float32 uint8)
# for y in range(0,2*half_chunk_size):
# for x in range(0,2*half_chunk_size):
# yt = y - half_chunk_size
# xt = x - half_chunk_size
# r = int( math.sqrt( yt*yt + xt*xt ) )
# if r > half_chunk_size-1:
# r = half_chunk_size-1
# kv = half_kernel_1d[r]
# alpha[ y, x, 0] = alpha[ y, x, 1] = alpha[ y, x, 2] = kv
# # Grab the indices of pixels along the projected image border and blend into the
# # background with a gaussian blur and alpha map.
# dst_mask_y_border = np.concatenate( [ dst_mask_y[ 0:,0 ],
# dst_mask_y[ 0:, -1 ],
# dst_mask_y[ 0, 0: ],
# dst_mask_y[-1, 0: ] ] )
# dst_mask_x_border = np.concatenate( [ dst_mask_x[ 0:,0 ],
# dst_mask_x[ 0:, -1 ],
# dst_mask_x[ 0, 0: ],
# dst_mask_x[-1, 0: ] ] )
# dst_mask_border = np.column_stack( [ dst_mask_y_border, dst_mask_x_border ] )
# #for y_ind in np.arange( 0, dst_mask_y.shape[0], 75 ):
# for border_pix in dst_mask_border[::75]:
# border_y = border_pix[0] #dst_mask_y[y_ind,0]
# border_x = border_pix[1] #dst_mask_x[y_ind,0]
# sub_img = mosaic_img[ border_y - half_size : border_y + half_size, border_x - half_size : border_x + half_size ].copy()
# sub_rng = border + 2*half_chunk_size
# sub_img[border:sub_rng,border:sub_rng] = cv2.GaussianBlur( sub_img[border:sub_rng,border:sub_rng], (81,81), 0 )
# mosaic_img[ border_y - half_chunk_size : border_y + half_chunk_size, border_x - half_chunk_size : border_x + half_chunk_size ] = \
# np.multiply( sub_img[border:sub_rng,border:sub_rng].astype( np.float32 ), alpha ) + \
# np.multiply( mosaic_img[ border_y - half_chunk_size : border_y + half_chunk_size, border_x - half_chunk_size : border_x + half_chunk_size ].astype( np.float32 ), 1 - alpha )
#cv2.imwrite('c:\\alpha.png', alpha)
#mosaic_img[ border_y - half_chunk_size : border_y + half_chunk_size, border_x - half_chunk_size : border_x + half_chunk_size ] = alpha
#mosaic_img[ border_y - half_chunk_size : border_y + half_chunk_size, border_x - half_chunk_size : border_x + half_chunk_size ] = sub_img[border:sub_rng,border:sub_rng]
elif interp_mode == 'nearest':
# Implementing nearest this way rather than just changing the interpolation function of cv2.remap above
# will be more efficient because we'll avoid the reprojection back to the source image and sample it directly
# using our index mask.
dst_pixels_denormalized = features.denormalized_image_coordinates( dst_pixels, camera.width, camera.height )
# Create a full equirectangular index image with all zero indices for x and y
fdst_y, fdst_x = np.zeros( ( 2, camera.height, camera.width ) ).astype( np.float32 )
# Use the projected indices to swap in the source image indices.
x = dst_pixels_denormalized[..., 0].astype(np.int32)
y = dst_pixels_denormalized[..., 1].astype(np.int32)
fdst_x[ y, x ] = src_pixels_denormalized[...,0]
fdst_y[ y, x ] = src_pixels_denormalized[...,1]
r_map_x = fdst_x
r_map_y = fdst_y
mosaic_img = cv2.remap( img, fdst_x, fdst_y, cv2.INTER_NEAREST, borderMode=cv2.BORDER_CONSTANT )
else:
raise NotImplementedError( 'Interpolation type not supported: {}'.format( interp_mode ) )
data.save_full_mosaic_image( os.path.splitext( shot.id )[0], mosaic_img )
end = timer()
report = {
"image": shot.id,
"wall_time": end - start,
}
data.save_report( io.json_dumps(report),
'full_mosaic_reprojection/{}.json'.format( shot.id ) )
return ( r_map_x, r_map_y, dst_mask_x, dst_mask_y )
def label_pointcloud(data_path, semantics_path):
#We take each shot, see which points fall on it and label it with corresponding semantic label.
data = dataset.DataSet(data_path)
reconstruction = data.load_reconstruction()[0]
tracks_manager = data.load_tracks_manager()
images = tracks_manager.get_shot_ids()
imcount = 0
ptcount = 0
vote_dict = {}
campose = np.empty((0, 3))
for im in images:
if (not (data.load_exif(im)['camera'] in reconstruction.cameras)):
continue
camera = reconstruction.cameras[data.load_exif(im)['camera']]
# print("Camera W, H ", camera.width, camera.height)
pts2d, pts3d = get_shot_observations(tracks_manager, reconstruction,
camera, im)
if (not (im in reconstruction.shots)):
print(im, " not in shots!")
continue
shot = reconstruction.shots[im]
campose = np.vstack((campose, shot.pose.translation))
semantic_image_path = os.path.join(semantics_path,
os.path.splitext(im)[0] + ".png")
semantic_image = cv2.imread(semantic_image_path)
print("Getting projections for image: ", im)
#TODO: Filter outliers by reprojection threshold?
#TODO: Create walkable area by filling up non object shit.
pointcloud = np.empty((0, 3))
for i in range(len(pts3d)):
# inlier = testInlier(pts3d[i], pts2d[i], camera, shot, 0.004) #resection_threshold from config.py
# if(not(inlier)):
# continue
pt2d = features.denormalized_image_coordinates(
np.array(pts2d[i]).reshape(-1, 2), semantic_image.shape[1],
semantic_image.shape[0])[0]
# print(pt2d)
row = int(pt2d[1])
col = int(pt2d[0])
# print(row, col)
point = pts3d[i]
pointcloud = np.vstack((pointcloud, point))
if (not (point.tobytes() in vote_dict)):
vote_dict[point.tobytes()] = []
rgb = semantic_image[row, col]
if (np.all(rgb == np.array([98, 98, 98]))):
rgb = 1
elif (np.all(rgb == np.array([100, 100, 100]))):
rgb = 1
else:
rgb = 2
vote_dict[point.tobytes()].append(rgb)
ptcount = ptcount + 1
imcount = imcount + 1
pcl, groundPoints = get_pointcloud_from_vote(vote_dict)
print(groundPoints)
#Fill up the non-walkable area.
planePointSet = pcl[:12000, :3]
# if(groundPoints.shape[0] > 2):
# planePointSet = groundPoints
boundMin = np.amin(planePointSet, axis=0)
boundMax = np.amax(planePointSet, axis=0)
planePtList = samplePlanePoints(None, boundMin, boundMax)
print("Sampled ", len(planePtList), " points on plane")
validPlanePts = np.empty((0, 6))
count = 0
n_elem = len(planePtList)
interval = int(len(planePtList) / n_elem)
planePtList = planePtList[::interval]
for planePt in planePtList:
color = getPointColorFromReconstruction(reconstruction, semantics_path,
planePt)
if (color is None):
continue
validPt = np.hstack((planePt, color))
validPlanePts = np.vstack((validPlanePts, validPt))
labelCompletionPercent = int(count / len(planePtList) * 100)
if (count % 20 == 0):
print("Completed ", labelCompletionPercent, "% of labeling")
count = count + 1
final_pcl = np.vstack((pcl[:12000, :], validPlanePts))
print("We got ", validPlanePts.shape[0], " valid plane points")
# final_pcl = pcl[:12000, :]
print("Labeled ", ptcount, " points")
print("Labeled for ", imcount, " images")
flat = flatten_by_plane_proj(final_pcl[:, :3], np.array([0, 0, 1, 0]),
(640, 480), final_pcl[:, 3:])
#Enable to Plot CAMERA POSE on IMAGE.
# for pt in campose:
# ptproj = flatten_coords_by_plane_proj(pt, final_pcl[:, :3], np.array([0, 0, 1, 0]), (640, 480))
# flat[ptproj[0, 1], ptproj[0, 0], :] = np.array([0, 0, 255])
cv2.imwrite("flattened_img.jpg", flat)
return final_pcl
def pix_coords(x, image):
return features.denormalized_image_coordinates(np.array([[x[0], x[1]]]),
image.shape[1],
image.shape[0])[0]
