def camera_from_json(key, obj):
"""
Read camera from a json object
"""
pt = obj.get('projection_type', 'perspective')
if pt == 'perspective':
camera = types.PerspectiveCamera()
camera.id = key
camera.width = obj.get('width', 0)
camera.height = obj.get('height', 0)
camera.focal = obj['focal']
camera.k1 = obj.get('k1', 0.0)
camera.k2 = obj.get('k2', 0.0)
camera.focal_prior = obj.get('focal_prior', camera.focal)
camera.k1_prior = obj.get('k1_prior', camera.k1)
camera.k2_prior = obj.get('k2_prior', camera.k2)
return camera
elif pt in ['equirectangular', 'spherical']:
camera = types.SphericalCamera()
camera.id = key
camera.width = obj['width']
camera.height = obj['height']
return camera
else:
raise NotImplementedError
def camera_from_exif_metadata(metadata, data):
'''
Create a camera object from exif metadata
'''
pt = metadata.get('projection_type', 'perspective')
if pt == 'perspective':
calib = (hard_coded_calibration(metadata)
or focal_ratio_calibration(metadata)
or default_calibration(data))
camera = types.PerspectiveCamera()
camera.id = metadata['camera']
camera.width = metadata['width']
camera.height = metadata['height']
camera.projection_type = metadata.get('projection_type', 'perspective')
camera.focal = calib['focal']
camera.k1 = calib['k1']
camera.k2 = calib['k2']
camera.focal_prior = calib['focal']
camera.k1_prior = calib['k1']
camera.k2_prior = calib['k2']
return camera
elif pt in ['equirectangular', 'spherical']:
camera = types.SphericalCamera()
camera.id = metadata['camera']
camera.width = metadata['width']
camera.height = metadata['height']
return camera
else:
raise ValueError("Unknown projection type: {}".format(pt))
def camera_from_exif_metadata(metadata, data):
'''
Create a camera object from exif metadata
'''
pt = metadata.get('projection_type', 'perspective').lower()
if pt == 'perspective':
calib = (hard_coded_calibration(metadata)
or focal_ratio_calibration(metadata)
or default_calibration(data))
camera = types.PerspectiveCamera()
camera.id = metadata['camera']
camera.width = metadata['width']
camera.height = metadata['height']
camera.projection_type = pt
camera.focal = calib['focal']
camera.k1 = calib['k1']
camera.k2 = calib['k2']
return camera
elif pt == 'brown':
calib = (hard_coded_calibration(metadata)
or focal_xy_calibration(metadata)
or default_calibration(data))
camera = types.BrownPerspectiveCamera()
camera.id = metadata['camera']
camera.width = metadata['width']
camera.height = metadata['height']
camera.projection_type = pt
camera.focal_x = calib['focal_x']
camera.focal_y = calib['focal_y']
camera.c_x = calib['c_x']
camera.c_y = calib['c_y']
camera.k1 = calib['k1']
camera.k2 = calib['k2']
camera.p1 = calib['p1']
camera.p2 = calib['p2']
camera.k3 = calib['k3']
return camera
elif pt == 'fisheye':
calib = (hard_coded_calibration(metadata)
or focal_ratio_calibration(metadata)
or default_calibration(data))
camera = types.FisheyeCamera()
camera.id = metadata['camera']
camera.width = metadata['width']
camera.height = metadata['height']
camera.projection_type = pt
camera.focal = calib['focal']
camera.k1 = calib['k1']
camera.k2 = calib['k2']
return camera
elif pt in ['equirectangular', 'spherical']:
camera = types.SphericalCamera()
camera.id = metadata['camera']
camera.width = metadata['width']
camera.height = metadata['height']
return camera
else:
raise ValueError("Unknown projection type: {}".format(pt))
def test_spherical_camera_projection():
"""Test spherical projection--backprojection loop."""
camera = types.SphericalCamera()
camera.width = 800
camera.height = 600
pixel = [0.1, 0.2]
bearing = camera.pixel_bearing(pixel)
projected = camera.project(bearing)
assert np.allclose(pixel, projected)
def camera_from_json(key, obj):
"""
Read camera from a json object
"""
pt = obj.get('projection_type', 'perspective')
if pt == 'perspective':
camera = types.PerspectiveCamera()
camera.id = key
camera.width = obj.get('width', 0)
camera.height = obj.get('height', 0)
camera.focal = obj['focal']
camera.k1 = obj.get('k1', 0.0)
camera.k2 = obj.get('k2', 0.0)
return camera
if pt == 'brown':
camera = types.BrownPerspectiveCamera()
camera.id = key
camera.width = obj.get('width', 0)
camera.height = obj.get('height', 0)
camera.focal_x = obj['focal_x']
camera.focal_y = obj['focal_y']
camera.c_x = obj.get('c_x', 0.0)
camera.c_y = obj.get('c_y', 0.0)
camera.k1 = obj.get('k1', 0.0)
camera.k2 = obj.get('k2', 0.0)
camera.p1 = obj.get('p1', 0.0)
camera.p2 = obj.get('p2', 0.0)
camera.k3 = obj.get('k3', 0.0)
return camera
elif pt == 'fisheye':
camera = types.FisheyeCamera()
camera.id = key
camera.width = obj.get('width', 0)
camera.height = obj.get('height', 0)
camera.focal = obj['focal']
camera.k1 = obj.get('k1', 0.0)
camera.k2 = obj.get('k2', 0.0)
return camera
elif pt == 'dual':
camera = types.DualCamera()
camera.id = key
camera.width = obj.get('width', 0)
camera.height = obj.get('height', 0)
camera.focal = obj['focal']
camera.k1 = obj.get('k1', 0.0)
camera.k2 = obj.get('k2', 0.0)
camera.transition = obj.get('transition', 0.5)
return camera
elif pt in ['equirectangular', 'spherical']:
camera = types.SphericalCamera()
camera.id = key
camera.width = obj['width']
camera.height = obj['height']
return camera
else:
raise NotImplementedError
def convert_points(self, p_unsorted, f_unsorted, c_unsorted):
image_camera_model = types.SphericalCamera()
image_camera_model.id = "v2 nctech pulsar 11000 5500 equirectangular 0.1666"
image_camera_model.width = 11000
image_camera_model.height = 5500
if len(p_unsorted) > 0:
# Mask pixels that are out of valid image bounds before converting to equirectangular image coordinates
bearings = image_camera_model.unfolded_pixel_bearings(
p_unsorted[:, :2])
p_mask = np.array([point is not None for point in bearings])
p_unsorted = p_unsorted[p_mask]
f_unsorted = f_unsorted[p_mask]
c_unsorted = c_unsorted[p_mask]
p_unsorted[:, :
2] = self.unfolded_cube_to_equi_normalized_image_coordinates(
p_unsorted[:, :2], image_camera_model)
return p_unsorted, f_unsorted, c_unsorted
def _get_spherical_camera():
camera = types.SphericalCamera()
camera.width = 800
camera.height = 600
return camera
def _get_spherical_camera():
camera = types.SphericalCamera()
camera.width = 800
camera.height = 600
camera_cpp = pygeometry.Camera.create_spherical()
return camera, camera_cpp
def convert_image(self, image, img, max_size):
image_camera_model = types.SphericalCamera()
image_camera_model.id = "v2 nctech pulsar 11000 5500 equirectangular 0.1666"
image_camera_model.width = 11000
image_camera_model.height = 5500
undist_tile_size = max_size // 4
undist_img = np.zeros((max_size // 2, max_size, 3), np.uint8)
undist_mask = np.full((max_size // 2, max_size, 1), 255, np.uint8)
undist_mask[undist_tile_size:2 * undist_tile_size,
2 * undist_tile_size:3 * undist_tile_size] = 0
undist_mask[undist_tile_size:2 * undist_tile_size,
undist_tile_size:2 * undist_tile_size] = 0
spherical_shot = types.Shot()
spherical_shot.pose = types.Pose()
spherical_shot.id = image
spherical_shot.camera = image_camera_model
perspective_shots = undistort.perspective_views_of_a_panorama(
spherical_shot, undist_tile_size)
for subshot in perspective_shots:
undistorted = undistort.render_perspective_view_of_a_panorama(
img, spherical_shot, subshot)
subshot_id_prefix = '{}_perspective_view_'.format(
spherical_shot.id)
subshot_name = subshot.id[
len(subshot_id_prefix):] if subshot.id.startswith(
subshot_id_prefix) else subshot.id
(subshot_name, ext) = os.path.splitext(subshot_name)
if subshot_name == 'front':
undist_img[:undist_tile_size, :undist_tile_size] = undistorted
# print( 'front')
elif subshot_name == 'left':
undist_img[:undist_tile_size,
undist_tile_size:2 * undist_tile_size] = undistorted
# print( 'left')
elif subshot_name == 'back':
undist_img[:undist_tile_size, 2 * undist_tile_size:3 *
undist_tile_size] = undistorted
# print( 'back')
elif subshot_name == 'right':
undist_img[:undist_tile_size, 3 * undist_tile_size:4 *
undist_tile_size] = undistorted
# print( 'right')
elif subshot_name == 'top':
undist_img[undist_tile_size:2 * undist_tile_size, 3 *
undist_tile_size:4 * undist_tile_size] = undistorted
# print( 'top')
elif subshot_name == 'bottom':
undist_img[undist_tile_size:2 *
undist_tile_size, :undist_tile_size] = undistorted
# print( 'bottom')
# data.save_undistorted_image(subshot.id, undist_img)
return undist_img
def create_full_mosaic(args):
log.setup()
shot, data = args
logger.info('Creating full mosaic for image {}'.format( shot.id ) )
config = data.config
start = timer()
projection_type = shot.camera.projection_type
r_map_x = None
r_map_y = None
dst_mask_x = None
dst_mask_y = None
if projection_type in ['perspective', 'brown', 'fisheye']:
img = data.load_image( shot.id )
camera = types.SphericalCamera()
camera.id = "Spherical Projection Camera"
# Determine the correct mosaic size from the focal length of the camera
# Limit this to a maximum of a 16K image which is the highest resolution
# currently supported by PVR.
K_pix = shot.camera.get_K_in_pixel_coordinates()
camera.height = int( np.clip( math.pi*K_pix[0,0], 0, 8192 ) )
camera.width = int( np.clip( 2*math.pi*K_pix[0,0], 0, 16384 ) )
shot_cam = shot.camera
# Project shot's pixels to the spherical mosaic image
src_shape = ( shot_cam.height, shot_cam.width )
src_y, src_x = np.indices( src_shape ).astype( np.float32 )
src_pixels_denormalized = np.column_stack( [ src_x.ravel(), src_y.ravel() ] )
src_pixels = features.normalized_image_coordinates( src_pixels_denormalized, shot_cam.width, shot_cam.height )
# Convert to bearings
src_bearings = shot_cam.pixel_bearing_many( src_pixels )
# Project to spherical mosaic pixels
dst_x, dst_y = camera.project( ( src_bearings[:, 0],
src_bearings[:, 1],
src_bearings[:, 2] ) )
dst_pixels = np.column_stack( [ dst_x.ravel(), dst_y.ravel() ] )
interp_mode = data.config.get( 'full_mosaic_proj_interpolation', 'linear' )
if interp_mode == 'linear':
# Snap to pixel centers to generate a projection index mask. This will be slower then finding
# the ROI using the projected border but it's far easier and covers wrap around and the poles with
# minimal effort. It will also probably be more efficient when wrap around or crossing the poles does occur.
dst_pixels_denormalized_int = features.denormalized_image_coordinates( dst_pixels, camera.width, camera.height ).astype( np.int32 )
dst_pixels_snap = features.normalized_image_coordinates( dst_pixels_denormalized_int.astype( np.float32 ), camera.width, camera.height )
dst_bearings_re = camera.pixel_bearing_many( dst_pixels_snap )
# Project mosaic pixel center bearings back into the source image
src_re_x, src_re_y = shot_cam.project( ( dst_bearings_re[:, 0],
dst_bearings_re[:, 1],
dst_bearings_re[:, 2] ) )
src_re_pixels = np.column_stack( [ src_re_x.ravel(), src_re_y.ravel() ] )
src_re_denormalized = features.denormalized_image_coordinates( src_re_pixels, shot_cam.width, shot_cam.height )
mosaic_img = initialize_mosaic_image( camera.width, camera.height, img )
# Reshape arrays for cv.remap efficiency reasons and due to the SHRT_MAX limit of array size.
# Another option is to process in chunks of linear array of shize SHRT_MAX. However, this
# approach was probably 4x slower.
x = src_re_denormalized[:, 0].reshape( src_x.shape ).astype(np.float32)
y = src_re_denormalized[:, 1].reshape( src_y.shape ).astype(np.float32)
r_map_x = x
r_map_y = y
# Sample source imagery colors
colors = cv2.remap( img, x, y, cv2.INTER_LINEAR , borderMode=cv2.BORDER_CONSTANT )
dst_mask_y = dst_pixels_denormalized_int[:, 1].reshape( src_y.shape )
dst_mask_x = dst_pixels_denormalized_int[:, 0].reshape( src_x.shape )
mosaic_img[ dst_mask_y, dst_mask_x ] = colors
blend_projection_border( mosaic_img, dst_mask_y, dst_mask_x )
# Initialize blurring and alpha mask kernels
# half_chunk_size = 75
# border = 41
# half_size = half_chunk_size + border
# kernel_1d = cv2.getGaussianKernel( 2*half_chunk_size+1, 1.5*(0.3*((2*half_chunk_size+1-1)*0.5 - 1) + 0.8) , cv2.CV_32F )
# kernel_1d/=kernel_1d[ half_chunk_size ]
# half_kernel_1d = kernel_1d[ half_chunk_size : 2*half_chunk_size ]
# alpha = np.zeros( ( 2*half_chunk_size, 2*half_chunk_size, 3 ), dtype = np.float32 ) #np.float32 uint8)
# for y in range(0,2*half_chunk_size):
# for x in range(0,2*half_chunk_size):
# yt = y - half_chunk_size
# xt = x - half_chunk_size
# r = int( math.sqrt( yt*yt + xt*xt ) )
# if r > half_chunk_size-1:
# r = half_chunk_size-1
# kv = half_kernel_1d[r]
# alpha[ y, x, 0] = alpha[ y, x, 1] = alpha[ y, x, 2] = kv
# # Grab the indices of pixels along the projected image border and blend into the
# # background with a gaussian blur and alpha map.
# dst_mask_y_border = np.concatenate( [ dst_mask_y[ 0:,0 ],
# dst_mask_y[ 0:, -1 ],
# dst_mask_y[ 0, 0: ],
# dst_mask_y[-1, 0: ] ] )
# dst_mask_x_border = np.concatenate( [ dst_mask_x[ 0:,0 ],
# dst_mask_x[ 0:, -1 ],
# dst_mask_x[ 0, 0: ],
# dst_mask_x[-1, 0: ] ] )
# dst_mask_border = np.column_stack( [ dst_mask_y_border, dst_mask_x_border ] )
# #for y_ind in np.arange( 0, dst_mask_y.shape[0], 75 ):
# for border_pix in dst_mask_border[::75]:
# border_y = border_pix[0] #dst_mask_y[y_ind,0]
# border_x = border_pix[1] #dst_mask_x[y_ind,0]
# sub_img = mosaic_img[ border_y - half_size : border_y + half_size, border_x - half_size : border_x + half_size ].copy()
# sub_rng = border + 2*half_chunk_size
# sub_img[border:sub_rng,border:sub_rng] = cv2.GaussianBlur( sub_img[border:sub_rng,border:sub_rng], (81,81), 0 )
# mosaic_img[ border_y - half_chunk_size : border_y + half_chunk_size, border_x - half_chunk_size : border_x + half_chunk_size ] = \
# np.multiply( sub_img[border:sub_rng,border:sub_rng].astype( np.float32 ), alpha ) + \
# np.multiply( mosaic_img[ border_y - half_chunk_size : border_y + half_chunk_size, border_x - half_chunk_size : border_x + half_chunk_size ].astype( np.float32 ), 1 - alpha )
#cv2.imwrite('c:\\alpha.png', alpha)
#mosaic_img[ border_y - half_chunk_size : border_y + half_chunk_size, border_x - half_chunk_size : border_x + half_chunk_size ] = alpha
#mosaic_img[ border_y - half_chunk_size : border_y + half_chunk_size, border_x - half_chunk_size : border_x + half_chunk_size ] = sub_img[border:sub_rng,border:sub_rng]
elif interp_mode == 'nearest':
# Implementing nearest this way rather than just changing the interpolation function of cv2.remap above
# will be more efficient because we'll avoid the reprojection back to the source image and sample it directly
# using our index mask.
dst_pixels_denormalized = features.denormalized_image_coordinates( dst_pixels, camera.width, camera.height )
# Create a full equirectangular index image with all zero indices for x and y
fdst_y, fdst_x = np.zeros( ( 2, camera.height, camera.width ) ).astype( np.float32 )
# Use the projected indices to swap in the source image indices.
x = dst_pixels_denormalized[..., 0].astype(np.int32)
y = dst_pixels_denormalized[..., 1].astype(np.int32)
fdst_x[ y, x ] = src_pixels_denormalized[...,0]
fdst_y[ y, x ] = src_pixels_denormalized[...,1]
r_map_x = fdst_x
r_map_y = fdst_y
mosaic_img = cv2.remap( img, fdst_x, fdst_y, cv2.INTER_NEAREST, borderMode=cv2.BORDER_CONSTANT )
else:
raise NotImplementedError( 'Interpolation type not supported: {}'.format( interp_mode ) )
data.save_full_mosaic_image( os.path.splitext( shot.id )[0], mosaic_img )
end = timer()
report = {
"image": shot.id,
"wall_time": end - start,
}
data.save_report( io.json_dumps(report),
'full_mosaic_reprojection/{}.json'.format( shot.id ) )
return ( r_map_x, r_map_y, dst_mask_x, dst_mask_y )
def create_full_mosaic_precom_maps( args ):
log.setup()
shot, r_map_x, r_map_y, dst_mask_x, dst_mask_y, data = args
logger.info('Creating full mosaic for image {}'.format( shot.id ) )
config = data.config
start = timer()
projection_type = shot.camera.projection_type
if projection_type in ['perspective', 'brown', 'fisheye']:
img = data.load_image( shot.id )
camera = types.SphericalCamera()
camera.id = "Spherical Projection Camera"
# Determine the correct mosaic size from the focal length of the camera
# Limit this to a maximum of a 16K image which is the highest resolution
# currently supported by PVR.
K_pix = shot.camera.get_K_in_pixel_coordinates()
camera.height = int( np.clip( math.pi*K_pix[0,0], 0, 8192 ) )
camera.width = int( np.clip( 2*math.pi*K_pix[0,0], 0, 16384 ) )
interp_mode = data.config.get( 'full_mosaic_proj_interpolation', 'linear' )
if interp_mode == 'linear':
mosaic_img = initialize_mosaic_image( camera.width, camera.height, img )
# Sample source imagery colors
colors = cv2.remap( img, r_map_x, r_map_y, cv2.INTER_LINEAR , borderMode=cv2.BORDER_CONSTANT )
mosaic_img[ dst_mask_y, dst_mask_x ] = colors
blend_projection_border( mosaic_img, dst_mask_y, dst_mask_x )
elif interp_mode == 'nearest':
mosaic_img = cv2.remap( img, r_map_x, r_map_y, cv2.INTER_NEAREST, borderMode=cv2.BORDER_CONSTANT )
else:
raise NotImplementedError( 'Interpolation type not supported: {}'.format( interp_mode ) )
data.save_full_mosaic_image( os.path.splitext( shot.id )[0], mosaic_img )
end = timer()
report = {
"image": shot.id,
"wall_time": end - start,
}
data.save_report( io.json_dumps(report),
'full_mosaic_reprojection/{}.json'.format( shot.id ) )
