def __init__(self, batch_size, num_classes, training=True, normalize=None):
self._num_classes = num_classes
self.training = training
self.normalize = normalize
self.batch_size = batch_size
self.data_path = "/home/fengjia/data/sets/nuscenes"
self.nusc= NuScenes(version='v1.0-trainval', dataroot = self.data_path, verbose= True)
self.explorer = NuScenesExplorer(self.nusc)
self.classes = ('__background__',
'pedestrian', 'barrier', 'trafficcone', 'bicycle', 'bus', 'car', 'construction', 'motorcycle', 'trailer', 'truck')
# PATH = self.data_path + '/annotations_list.txt'
PATH = self.data_path + '/car_pedestrian_annotations_list.txt'
with open(PATH) as f:
self.token = [x.strip() for x in f.readlines()]
self.token = self.token[:400]
def get_pcl():
nusc = NuScenes(version='v1.0-mini', dataroot='/data/sets/nuscenes', verbose=True)
explorer = NuScenesExplorer(nusc)
imglist = []
count = 0
for scene in nusc.scene:
token = scene['first_sample_token']
while token != '':
count += 1
sample = nusc.get('sample',token)
sample_data_cam = nusc.get('sample_data', sample['data']['CAM_FRONT'])
sample_data_pcl = nusc.get('sample_data', sample['data']['LIDAR_TOP'])
# get CAM_FRONT datapath
data_path, boxes, camera_intrinsic = explorer.nusc.get_sample_data(sample_data_cam['token'], box_vis_level = BoxVisibility.ANY)
imglist += [data_path]
# get LiDAR point cloud
sample_rec = explorer.nusc.get('sample', sample_data_pcl['sample_token'])
chan = sample_data_pcl['channel']
ref_chan = 'LIDAR_TOP'
pc, times = LidarPointCloud.from_file_multisweep(nusc, sample_rec, chan, ref_chan)
radius = np.sqrt((pc.points[0])**2 + (pc.points[1])**2 + (pc.points[2])**2)
pc = pc.points.transpose()
pc = pc[np.where(radius<20)]
print(pc)
display(pc.transpose(), count)
token = sample['next']
if count == 2:
break
if count == 2:
break
plt.show()
print(len(imglist))
import torch
import torch.nn as nn
import torch.nn.parallel
import torch.utils.data
from torch.autograd import Variable
import torch.nn.functional as F
import torch.nn.init as init
from model.pointnet import PointNetfeat
import torch
nusc= NuScenes(version='v1.0-trainval', dataroot = '/data/sets/nuscenes', verbose= True)
explorer = NuScenesExplorer(nusc)
"""
class InputTransformNet(nn.Module):
def __init__(self):
super(InputTransformNet, self).__init__()
self.relu = nn.ReLU()
self.conv1 = nn.Conv1d(3, 64, 1)
self.bn1 = nn.BatchNorm1d(64)
self.conv2 = nn.Conv1d(64, 128, 1)
self.bn2 = nn.BatchNorm1d(128)
self.conv3 = nn.Conv1d(128, 1024, 1)
self.bn3 = nn.BatchNorm1d(1024)
self.fc1 = nn.Linear(1024, 512)
self.bn4 = nn.BatchNorm1d(512)
self.fc2 = nn.Linear(512, 256)
return points, coloring, im
if __name__ == '__main__':
data_path = r"/home/fengjia/data/sets/nuscenes"
# save_path = r"/home/fengjia/data/sets/nuscenes_local/vehicle"
save_path = r"/home/fengjia/data/sets/nuscenes_temp/vehicle"
if not os.path.isdir(save_path):
print('save path not exist')
exit(0)
nusc = NuScenes(version='v1.0-trainval', dataroot=data_path, verbose=True)
explorer = NuScenesExplorer(nusc)
PATH = data_path + '/CAMFRONT.txt'
with open(PATH) as f:
image_token = [x.strip() for x in f.readlines()]
annotations = []
counter = 0
max_v = 0
min_v = 999999
# pdb.set_trace()
for im_token in image_token:
sample_data = nusc.get('sample_data', im_token)
image_name = sample_data['filename']
img = cv2.imread('/home/fengjia/data/sets/nuscenes/' + image_name)
def get_pointcloud(nusc,
bottom_left,
top_right,
box,
pointsensor_token: str,
camera_token: str,
min_dist: float = 1.0) -> Tuple:
"""
Given a point sensor (lidar/radar) token and camera sample_data token, load point-cloud and map it to the image
plane.
:param pointsensor_token: Lidar/radar sample_data token.
:param camera_token: Camera sample_data token.
:param min_dist: Distance from the camera below which points are discarded.
:return (pointcloud <np.float: 2, n)>, coloring <np.float: n>, image <Image>).
"""
sample_data = nusc.get("sample_data", camera_token)
explorer = NuScenesExplorer(nusc)
cam = nusc.get('sample_data', camera_token)
pointsensor = nusc.get('sample_data', pointsensor_token)
im = Image.open(osp.join(nusc.dataroot, cam['filename']))
sample_rec = explorer.nusc.get('sample', pointsensor['sample_token'])
chan = pointsensor['channel']
ref_chan = 'LIDAR_TOP'
pc, times = LidarPointCloud.from_file_multisweep(nusc,
sample_rec,
chan,
ref_chan,
nsweeps=10)
data_path, boxes, camera_intrinsic = nusc.get_sample_data(
pointsensor_token, selected_anntokens=[box.token])
pcl_box = boxes[0]
original_points = pc.points.copy()
# Points live in the point sensor frame. So they need to be transformed via global to the image plane.
# First step: transform the point-cloud to the ego vehicle frame for the timestamp of the sweep.
cs_record = nusc.get('calibrated_sensor',
pointsensor['calibrated_sensor_token'])
pc.rotate(Quaternion(cs_record['rotation']).rotation_matrix)
pc.translate(np.array(cs_record['translation']))
# Second step: transform to the global frame.
poserecord = nusc.get('ego_pose', pointsensor['ego_pose_token'])
pc.rotate(Quaternion(poserecord['rotation']).rotation_matrix)
pc.translate(np.array(poserecord['translation']))
# Third step: transform into the ego vehicle frame for the timestamp of the image.
poserecord = nusc.get('ego_pose', cam['ego_pose_token'])
pc.translate(-np.array(poserecord['translation']))
pc.rotate(Quaternion(poserecord['rotation']).rotation_matrix.T)
# Fourth step: transform into the camera.
cs_record = nusc.get('calibrated_sensor', cam['calibrated_sensor_token'])
pc.translate(-np.array(cs_record['translation']))
pc.rotate(Quaternion(cs_record['rotation']).rotation_matrix.T)
# Fifth step: actually take a "picture" of the point cloud.
depths = pc.points[2, :]
# Take the actual picture (matrix multiplication with camera-matrix + renormalization).
points = view_points(pc.points[:3, :],
np.array(cs_record['camera_intrinsic']),
normalize=True)
center = np.array([[box.center[0]], [box.center[1]], [box.center[2]]])
box_center = view_points(center,
np.array(cs_record['camera_intrinsic']),
normalize=True)
box_corners = box.corners()
# Remove points that are either outside or behind the camera. Leave a margin of 1 pixel for aesthetic reasons.
# Also make sure points are at least 1m in front of the camera to avoid seeing the lidar points on the camera
# casing for non-keyframes which are slightly out of sync.
mask = np.ones(depths.shape[0], dtype=bool)
mask = np.logical_and(mask, depths > min_dist)
mask = np.logical_and(mask, points[0, :] > 1)
mask = np.logical_and(mask, points[0, :] < im.size[0] - 1)
mask = np.logical_and(mask, points[1, :] > 1)
mask = np.logical_and(mask, points[1, :] < im.size[1] - 1)
original_points = original_points[:, mask]
points = points[:, mask]
points_2 = pc.points[:, mask]
crop_mask = np.ones(points.shape[1], dtype=bool)
crop_mask = np.logical_and(crop_mask, points[1, :] > bottom_left[1])
crop_mask = np.logical_and(crop_mask, points[1, :] < top_right[1])
crop_mask = np.logical_and(crop_mask, points[0, :] > bottom_left[0])
crop_mask = np.logical_and(crop_mask, points[0, :] < top_right[0])
original_points = original_points[:, crop_mask]
points = points[:, crop_mask]
points_3 = points_2[:, crop_mask]
image_center = np.asarray([((bottom_left[0] + top_right[0]) / 2),
((bottom_left[1] + top_right[1]) / 2), 1])
# rotate to make the ray passing through the image_center into the z axis
z_axis = np.linalg.lstsq(np.array(cs_record['camera_intrinsic']),
image_center,
rcond=None)[0]
v = np.asarray([z_axis[0], z_axis[1], z_axis[2]])
z = np.asarray([0., 0., 1.])
normal = np.cross(v, z)
theta = np.arccos(np.dot(v, z) / np.sqrt(v.dot(v)))
new_pts = []
new_corner = []
old_pts = []
points_3 = points_3[:3, :]
translate = np.dot(rotation_matrix(normal, theta), image_center)
for point in points_3.T:
new_pts = new_pts + [np.dot(rotation_matrix(normal, theta), point)]
for corner in box_corners.T:
new_corner = new_corner + [
np.dot(rotation_matrix(normal, theta), corner)
]
points = np.asarray(new_pts)
original_points = original_points[:3, :].T
new_corners = np.asarray(new_corner)
reverse_matrix = rotation_matrix(normal, -theta)
# Sample 400 points
if np.shape(new_pts)[0] > 400:
mask = np.random.choice(points.shape[0], 400, replace=False)
points = points[mask, :]
original_points = original_points[mask, :]
shift = np.expand_dims(np.mean(points, axis=0), 0)
points = points - shift # center
new_corners = new_corners - shift # center
dist = np.max(np.sqrt(np.sum(points**2, axis=1)), 0)
points = points / dist #scale
new_corners = new_corners / dist #scale
# Compute offset from point to corner
n = np.shape(points)[0]
offset = np.zeros((n, 8, 3))
for i in range(0, n):
for j in range(0, 8):
offset[i][j] = new_corners[j] - points[i]
# Compute mask on which points lie inside of the box
m = []
for point in original_points:
if in_box(point, pcl_box.corners()) == True:
m = m + [1]
else:
m = m + [0]
m = np.asarray(m)
return points.T, m, offset, reverse_matrix, new_corners