def __init__(self,
root_dataset,
ocam_dict,
pose_dict,
level,
reduction=2,
transform=None,
border_value=[256 // 2, 256 // 2, 256 // 2]):
self.root_dataset = root_dataset
self.ocam_dict = ocam_dict
self.pose_dict = pose_dict
self.level = level
self.transform = transform
# fisheye to ico
self.vertices, self.faces = get_icosahedron(level)
self.fisheye_mapxy_pad = {}
for key in self.ocam_dict:
self.fisheye_mapxy_pad[key] = mapxy_fisheye_to_ico(
self.ocam_dict[key], self.pose_dict[key], self.level)
# idepth to ico
self.idepth_level = level - reduction
self.idepth_mapxy_pad = self._mapxy_idepth_to_ico(self.idepth_level)
self.idepth_coord = get_unfold_imgcoord(self.idepth_level,
drop_NE=False)
# out of fov value
self.border_value = border_value
def calculate_weight(level=0):
""" calculate weight alpha
"""
img_coord = get_unfold_imgcoord(level, False)
vertices, _ = get_icosahedron(level)
key = 0
vi_idx = img_coord[key][1:, 1:]
vn1_idx = img_coord[key][1:, :-1]
vn6_idx = img_coord[key][:-1, 1:]
vi = vertices[vi_idx]
vn1 = vertices[vn1_idx]
vn6 = vertices[vn6_idx]
# vector from vi to north pole
north_pole = np.array([0, -1, 0])
to_north_pole = north_pole - vi
# unit vector from vi to neighbor
vn1vi = vn1 - vi
vn1vi /= norm(vn1vi, axis=-1, keepdims=True)
vn6vi = vn6 - vi
vn6vi /= norm(vn6vi, axis=-1, keepdims=True)
# face normal
face_n = np.cross(vn1vi, vn6vi)
face_n /= norm(face_n, axis=-1, keepdims=True)
# to north pole on tangent plane
proj_vec = np.sum(to_north_pole * face_n, axis=-1, keepdims=True) * face_n
np_tangent_plane = to_north_pole - proj_vec
np_tangent_plane /= norm(np_tangent_plane, axis=-1, keepdims=True)
# calculate cost
psi = np.arccos(np.sum(vn1vi * np_tangent_plane, axis=-1))
tmp_vals = np.sum(vn6vi * np_tangent_plane, axis=-1)
tmp_vals[0, 0] = np.clip(tmp_vals[0, 0], -1,
1) # make sure value is between -1 and 1
phi = np.arccos(tmp_vals)
base_weight = phi / (psi + phi)
weight = np.concatenate((np.flip(base_weight[-1:]), base_weight), axis=0)
weight = np.concatenate((np.flip(weight[:, -1:]), weight), axis=1)
# top_weight = base_weight[-1][::-1]
# left_weight = base_weight[:, -1][::-1]
# weight = np.zeros(img_coord[0].shape)
# weight[0, 0] = 0.5 # index 0 weight
# weight[1:, 1:] = base_weight
# weight[0, 1:] = top_weight
# weight[1:, 0] = left_weight
# to row weight
h, w = weight.shape
weight_row = 1 - np.concatenate((weight[:w], weight[w - 1:, 1:]), axis=1)
return weight, weight_row
def ico_to_erp_idx_weight(level, h=300, w=600):
xyz = uv2xyz(genuv(h, w))
pts_c = xyz.reshape(-1, 3)
v, f = get_icosahedron(level)
kdtree = cKDTree(v)
_, tmp_indices = kdtree.query(pts_c, k=4)
(v0_idx, v1_idx, v2_idx), (weight_0, weight_1,
weight_2) = weight_for_triangle_interpolation(
v, tmp_indices, pts_c)
return (v0_idx, v1_idx, v2_idx), (weight_0, weight_1, weight_2)
def _mapxy_idepth_to_ico(self, level):
v, f = get_icosahedron(level)
mapy, mapx = uv2img_idx(xyz2uv(v), h=320, w=640)
# reshape to prevent shrt_max error in remap
# https://answers.opencv.org/question/203798/shrt_max-in-cv2remap/
len_v = len(v)**0.5
tmp_h = int(len_v) + 1
pad_w = tmp_h * tmp_h - mapx.shape[0]
mapx = np.pad(mapx, (0, pad_w), 'constant', constant_values=-1)
mapy = np.pad(mapy, (0, pad_w), 'constant', constant_values=-1)
mapx = mapx.reshape(tmp_h, tmp_h).astype(np.float32)
mapy = mapy.reshape(tmp_h, tmp_h).astype(np.float32)
return mapx, mapy, pad_w
def __init__(self,
dataset,
erp_shape,
level,
rotate=False,
transform=None,
debug=False):
self.dataset = dataset
self.transform = transform
self.erp_shape = erp_shape
self.level = level
self.rotate = rotate
self.vertices, self.faces = get_icosahedron(level)
self.img_coord = get_unfold_imgcoord(level)
self.debug = debug
def icospherical_sweep_grid(Tcw_bytes, center_bytes, depth, level, fov=220, only_translation=False):
# numpy array is not hashable so give array as array.tobytes()
Tcw = np.frombuffer(Tcw_bytes).reshape(4, 4)
center = np.frombuffer(center_bytes)
# ------------------------------------
# find projected points on sphere
v, f = get_icosahedron(level)
pts_w = depth * v + center
# pt_c = T_cw*pt_w
if only_translation:
# dataloader deal with rotation
Twc = np.linalg.inv(Tcw)
pts_c = pts_w.T - Twc[:3, 3:4]
cam_dir = Twc[:3, :3].dot([0, 0, 1])
else:
pts_c = Tcw[:3, :3].dot(pts_w.T) + Tcw[:3, 3:4]
cam_dir = [0, 0, 1]
pts_c = pts_c.T
pts_c /= norm(pts_c, axis=1, keepdims=True)
# ------------------------------------
# extract inside fov
fov_rad = np.deg2rad(fov / 2)
cos_vals = pts_c.dot(cam_dir)
is_inside = np.cos(fov_rad) < cos_vals
# nearest neighbor search
kdtree = get_KDTree(level)
tmp_distances, tmp_indices = kdtree.query(pts_c[is_inside], k=4)
# tmp_distances, tmp_indices = kdtree.kneighbors(pts_c[is_inside])
# ------------------------------------
# calculate weight
vertex_ids, (weight_0, weight_1, weight_2) = weight_for_triangle_interpolation(v, tmp_indices, pts_c[is_inside])
# (v0_idx, v1_idx, v2_idx) = vertex_ids
weight_0 = torch.from_numpy(weight_0).float()
weight_1 = torch.from_numpy(weight_1).float()
weight_2 = torch.from_numpy(weight_2).float()
return vertex_ids, (weight_0, weight_1, weight_2), is_inside
def mapxy_fisheye_to_ico(ocam, T_wc=np.eye(4), level=0):
"""Map to project a fisheye image to icosahedron surface
Parameters
----------
ocam : OCamCamera (https://github.com/matsuren/ocamcalib_undistort)
OcamCalib camera class object
level : int
icosahedron resolution level
Examples
--------
>>> mapx, mapy, pad_w = mapxy_fisheye_to_ico(ocam, level)
>>> ico_img = cv2.remap(img, mapx, mapy,cv2.INTER_LINEAR)
>>> ico_img = ico_img.reshape(-1, 3)[:-pad_w]
"""
# get vertices
v, f = get_icosahedron(level)
# rotate vertices
T_cw = np.linalg.inv(T_wc)
rot_v = T_cw[:3, :3].dot(v.T)
mapx, mapy = ocam.world2cam(rot_v)
# reshape to prevent shrt_max error in remap
# https://answers.opencv.org/question/203798/shrt_max-in-cv2remap/
# from scipy.ndimage.interpolation import map_coordinates is slow
# so I chose to use opencv remap
len_v = len(v)**0.5
tmp_h = int(len_v) + 1
pad_w = tmp_h * tmp_h - mapx.shape[0]
mapx = np.pad(mapx, (0, pad_w), 'constant', constant_values=-1)
mapy = np.pad(mapy, (0, pad_w), 'constant', constant_values=-1)
mapx = mapx.reshape(tmp_h, tmp_h)
mapy = mapy.reshape(tmp_h, tmp_h)
return mapx, mapy, pad_w
def get_KDTree(level):
v, f = get_icosahedron(level)
return cKDTree(v)