def __init__(self, batch_size, num_classes, training=True, normalize=None): self._num_classes = num_classes self.training = training self.normalize = normalize self.batch_size = batch_size self.data_path = "/home/fengjia/data/sets/nuscenes" self.nusc= NuScenes(version='v1.0-trainval', dataroot = self.data_path, verbose= True) self.explorer = NuScenesExplorer(self.nusc) self.classes = ('__background__', 'pedestrian', 'barrier', 'trafficcone', 'bicycle', 'bus', 'car', 'construction', 'motorcycle', 'trailer', 'truck') # PATH = self.data_path + '/annotations_list.txt' PATH = self.data_path + '/car_pedestrian_annotations_list.txt' with open(PATH) as f: self.token = [x.strip() for x in f.readlines()] self.token = self.token[:400]
def get_pcl(): nusc = NuScenes(version='v1.0-mini', dataroot='/data/sets/nuscenes', verbose=True) explorer = NuScenesExplorer(nusc) imglist = [] count = 0 for scene in nusc.scene: token = scene['first_sample_token'] while token != '': count += 1 sample = nusc.get('sample',token) sample_data_cam = nusc.get('sample_data', sample['data']['CAM_FRONT']) sample_data_pcl = nusc.get('sample_data', sample['data']['LIDAR_TOP']) # get CAM_FRONT datapath data_path, boxes, camera_intrinsic = explorer.nusc.get_sample_data(sample_data_cam['token'], box_vis_level = BoxVisibility.ANY) imglist += [data_path] # get LiDAR point cloud sample_rec = explorer.nusc.get('sample', sample_data_pcl['sample_token']) chan = sample_data_pcl['channel'] ref_chan = 'LIDAR_TOP' pc, times = LidarPointCloud.from_file_multisweep(nusc, sample_rec, chan, ref_chan) radius = np.sqrt((pc.points[0])**2 + (pc.points[1])**2 + (pc.points[2])**2) pc = pc.points.transpose() pc = pc[np.where(radius<20)] print(pc) display(pc.transpose(), count) token = sample['next'] if count == 2: break if count == 2: break plt.show() print(len(imglist))
import torch import torch.nn as nn import torch.nn.parallel import torch.utils.data from torch.autograd import Variable import torch.nn.functional as F import torch.nn.init as init from model.pointnet import PointNetfeat import torch nusc= NuScenes(version='v1.0-trainval', dataroot = '/data/sets/nuscenes', verbose= True) explorer = NuScenesExplorer(nusc) """ class InputTransformNet(nn.Module): def __init__(self): super(InputTransformNet, self).__init__() self.relu = nn.ReLU() self.conv1 = nn.Conv1d(3, 64, 1) self.bn1 = nn.BatchNorm1d(64) self.conv2 = nn.Conv1d(64, 128, 1) self.bn2 = nn.BatchNorm1d(128) self.conv3 = nn.Conv1d(128, 1024, 1) self.bn3 = nn.BatchNorm1d(1024) self.fc1 = nn.Linear(1024, 512) self.bn4 = nn.BatchNorm1d(512) self.fc2 = nn.Linear(512, 256)
return points, coloring, im if __name__ == '__main__': data_path = r"/home/fengjia/data/sets/nuscenes" # save_path = r"/home/fengjia/data/sets/nuscenes_local/vehicle" save_path = r"/home/fengjia/data/sets/nuscenes_temp/vehicle" if not os.path.isdir(save_path): print('save path not exist') exit(0) nusc = NuScenes(version='v1.0-trainval', dataroot=data_path, verbose=True) explorer = NuScenesExplorer(nusc) PATH = data_path + '/CAMFRONT.txt' with open(PATH) as f: image_token = [x.strip() for x in f.readlines()] annotations = [] counter = 0 max_v = 0 min_v = 999999 # pdb.set_trace() for im_token in image_token: sample_data = nusc.get('sample_data', im_token) image_name = sample_data['filename'] img = cv2.imread('/home/fengjia/data/sets/nuscenes/' + image_name)
def get_pointcloud(nusc, bottom_left, top_right, box, pointsensor_token: str, camera_token: str, min_dist: float = 1.0) -> Tuple: """ Given a point sensor (lidar/radar) token and camera sample_data token, load point-cloud and map it to the image plane. :param pointsensor_token: Lidar/radar sample_data token. :param camera_token: Camera sample_data token. :param min_dist: Distance from the camera below which points are discarded. :return (pointcloud <np.float: 2, n)>, coloring <np.float: n>, image <Image>). """ sample_data = nusc.get("sample_data", camera_token) explorer = NuScenesExplorer(nusc) cam = nusc.get('sample_data', camera_token) pointsensor = nusc.get('sample_data', pointsensor_token) im = Image.open(osp.join(nusc.dataroot, cam['filename'])) sample_rec = explorer.nusc.get('sample', pointsensor['sample_token']) chan = pointsensor['channel'] ref_chan = 'LIDAR_TOP' pc, times = LidarPointCloud.from_file_multisweep(nusc, sample_rec, chan, ref_chan, nsweeps=10) data_path, boxes, camera_intrinsic = nusc.get_sample_data( pointsensor_token, selected_anntokens=[box.token]) pcl_box = boxes[0] original_points = pc.points.copy() # Points live in the point sensor frame. So they need to be transformed via global to the image plane. # First step: transform the point-cloud to the ego vehicle frame for the timestamp of the sweep. cs_record = nusc.get('calibrated_sensor', pointsensor['calibrated_sensor_token']) pc.rotate(Quaternion(cs_record['rotation']).rotation_matrix) pc.translate(np.array(cs_record['translation'])) # Second step: transform to the global frame. poserecord = nusc.get('ego_pose', pointsensor['ego_pose_token']) pc.rotate(Quaternion(poserecord['rotation']).rotation_matrix) pc.translate(np.array(poserecord['translation'])) # Third step: transform into the ego vehicle frame for the timestamp of the image. poserecord = nusc.get('ego_pose', cam['ego_pose_token']) pc.translate(-np.array(poserecord['translation'])) pc.rotate(Quaternion(poserecord['rotation']).rotation_matrix.T) # Fourth step: transform into the camera. cs_record = nusc.get('calibrated_sensor', cam['calibrated_sensor_token']) pc.translate(-np.array(cs_record['translation'])) pc.rotate(Quaternion(cs_record['rotation']).rotation_matrix.T) # Fifth step: actually take a "picture" of the point cloud. depths = pc.points[2, :] # Take the actual picture (matrix multiplication with camera-matrix + renormalization). points = view_points(pc.points[:3, :], np.array(cs_record['camera_intrinsic']), normalize=True) center = np.array([[box.center[0]], [box.center[1]], [box.center[2]]]) box_center = view_points(center, np.array(cs_record['camera_intrinsic']), normalize=True) box_corners = box.corners() # Remove points that are either outside or behind the camera. Leave a margin of 1 pixel for aesthetic reasons. # Also make sure points are at least 1m in front of the camera to avoid seeing the lidar points on the camera # casing for non-keyframes which are slightly out of sync. mask = np.ones(depths.shape[0], dtype=bool) mask = np.logical_and(mask, depths > min_dist) mask = np.logical_and(mask, points[0, :] > 1) mask = np.logical_and(mask, points[0, :] < im.size[0] - 1) mask = np.logical_and(mask, points[1, :] > 1) mask = np.logical_and(mask, points[1, :] < im.size[1] - 1) original_points = original_points[:, mask] points = points[:, mask] points_2 = pc.points[:, mask] crop_mask = np.ones(points.shape[1], dtype=bool) crop_mask = np.logical_and(crop_mask, points[1, :] > bottom_left[1]) crop_mask = np.logical_and(crop_mask, points[1, :] < top_right[1]) crop_mask = np.logical_and(crop_mask, points[0, :] > bottom_left[0]) crop_mask = np.logical_and(crop_mask, points[0, :] < top_right[0]) original_points = original_points[:, crop_mask] points = points[:, crop_mask] points_3 = points_2[:, crop_mask] image_center = np.asarray([((bottom_left[0] + top_right[0]) / 2), ((bottom_left[1] + top_right[1]) / 2), 1]) # rotate to make the ray passing through the image_center into the z axis z_axis = np.linalg.lstsq(np.array(cs_record['camera_intrinsic']), image_center, rcond=None)[0] v = np.asarray([z_axis[0], z_axis[1], z_axis[2]]) z = np.asarray([0., 0., 1.]) normal = np.cross(v, z) theta = np.arccos(np.dot(v, z) / np.sqrt(v.dot(v))) new_pts = [] new_corner = [] old_pts = [] points_3 = points_3[:3, :] translate = np.dot(rotation_matrix(normal, theta), image_center) for point in points_3.T: new_pts = new_pts + [np.dot(rotation_matrix(normal, theta), point)] for corner in box_corners.T: new_corner = new_corner + [ np.dot(rotation_matrix(normal, theta), corner) ] points = np.asarray(new_pts) original_points = original_points[:3, :].T new_corners = np.asarray(new_corner) reverse_matrix = rotation_matrix(normal, -theta) # Sample 400 points if np.shape(new_pts)[0] > 400: mask = np.random.choice(points.shape[0], 400, replace=False) points = points[mask, :] original_points = original_points[mask, :] shift = np.expand_dims(np.mean(points, axis=0), 0) points = points - shift # center new_corners = new_corners - shift # center dist = np.max(np.sqrt(np.sum(points**2, axis=1)), 0) points = points / dist #scale new_corners = new_corners / dist #scale # Compute offset from point to corner n = np.shape(points)[0] offset = np.zeros((n, 8, 3)) for i in range(0, n): for j in range(0, 8): offset[i][j] = new_corners[j] - points[i] # Compute mask on which points lie inside of the box m = [] for point in original_points: if in_box(point, pcl_box.corners()) == True: m = m + [1] else: m = m + [0] m = np.asarray(m) return points.T, m, offset, reverse_matrix, new_corners