def generateXYZRGB(ds: LyftDataset, sample_token: str, config: dict,
folder: str):
sample = ds.get("sample", sample_token)
sample_lidar_token = sample["data"]["LIDAR_TOP"]
lidar_data = ds.get("sample_data", sample_lidar_token)
lidar_filepath = ds.get_sample_data_path(sample_lidar_token)
ego_pose = ds.get("ego_pose", lidar_data["ego_pose_token"])
calibrated_sensor = ds.get("calibrated_sensor",
lidar_data["calibrated_sensor_token"])
global_from_car = transform_matrix(ego_pose['translation'],
Quaternion(ego_pose['rotation']),
inverse=False)
car_from_sensor = transform_matrix(calibrated_sensor['translation'],
Quaternion(
calibrated_sensor['rotation']),
inverse=False)
try:
lidar_pointcloud = LidarPointCloud.from_file(lidar_filepath)
pc = copy.deepcopy(lidar_pointcloud)
lidar_pointcloud.transform(car_from_sensor)
except Exception as e:
print("Failed to load Lidar Pointcloud for {}: {}:".format(
sample_token, e))
pass
XYZRGB = getXYZRGB(ds, pc, sample, lidar_data)
np.savez(folder + sample_lidar_token + ".xyzrgb")
def select_2d_annotation_boxes(ldf: LyftDataset, classifier, sample_token,
camera_type=['CAM_FRONT', 'CAM_BACK', 'CAM_FRONT_LEFT', 'CAM_FRONT_RIGHT',
'CAM_BACK_RIGHT', 'CAM_BACK_LEFT']) -> (str, str, np.ndarray):
sample_record = ldf.get('sample', sample_token)
cams = [key for key in sample_record["data"].keys() if "CAM" in key]
cams = [cam for cam in cams if cam in camera_type]
for cam in cams:
cam_token = sample_record["data"][cam]
image_file_path = ldf.get_sample_data_path(cam_token)
image_array = imread(image_file_path)
det_result = classifier.detect_multi_object(image_array, score_threshold=[0.4, 0.4, 0.4])
for i in range(det_result.shape[0]):
bounding_2d_box = det_result[i, 0:4]
score = det_result[i, 4]
class_idx = det_result[i, 5]
yield sample_token, cam_token, bounding_2d_box, score, class_idx
class LyftLEVEL5_utils():
def __init__(self, phase):
assert phase in ['train', 'test']
self.phase = phase
self.prep_list_of_sessions()
def load_PC(self, session_ind, cloud_ind, cache_file=None):
assert session_ind < self.num_sessions
assert cloud_ind < self.session_lengths[
session_ind], f"Requested cloud {cloud_ind}, but session {session_ind} only contains {self.session_lengths[session_ind]} clouds"
lidar_token = self.cloud_tokens[session_ind][cloud_ind]
cloud = self.load_cloud_raw(lidar_token)
if cache_file is not None:
np.save(cache_file, cloud)
return cloud
def get_relative_motion_A_to_B(self, session_ind, cloud_ind_A,
cloud_ind_B):
token_A = self.cloud_tokens[session_ind][cloud_ind_A]
posA = self.load_position_raw(token_A)
token_B = self.cloud_tokens[session_ind][cloud_ind_B]
posB = self.load_position_raw(token_B)
mot_A_to_B = np.linalg.inv(posB) @ posA
return mot_A_to_B
def prep_list_of_sessions(self):
self.level5data = LyftDataset(json_path=DATASET_ROOT +
"/%s_data" % self.phase,
data_path=DATASET_ROOT,
verbose=True)
self.num_sessions = len(self.level5data.scene)
self.cloud_tokens = []
self.session_lengths = []
for session_ind in range(self.num_sessions):
record = self.level5data.scene[session_ind]
session_token = record['token']
sample_token = record["first_sample_token"]
sample = self.level5data.get("sample", sample_token)
lidar_token = sample["data"]["LIDAR_TOP"]
cur_lidar_tokens = []
while len(lidar_token) > 0:
cur_lidar_tokens.append(lidar_token)
lidar_data = self.level5data.get("sample_data", lidar_token)
lidar_token = lidar_data["next"]
self.cloud_tokens.append(cur_lidar_tokens)
self.session_lengths.append(len(cur_lidar_tokens))
def load_position_raw(self, sample_lidar_token):
lidar_data = self.level5data.get("sample_data", sample_lidar_token)
ego_pose = self.level5data.get("ego_pose",
lidar_data["ego_pose_token"])
calibrated_sensor = self.level5data.get(
"calibrated_sensor", lidar_data["calibrated_sensor_token"])
# Homogeneous transformation matrix from car frame to world frame.
global_from_car = transform_matrix(ego_pose['translation'],
Quaternion(ego_pose['rotation']),
inverse=False)
return global_from_car
def load_cloud_raw(self, sample_lidar_token):
lidar_data = self.level5data.get("sample_data", sample_lidar_token)
lidar_filepath = self.level5data.get_sample_data_path(
sample_lidar_token)
ego_pose = self.level5data.get("ego_pose",
lidar_data["ego_pose_token"])
calibrated_sensor = self.level5data.get(
"calibrated_sensor", lidar_data["calibrated_sensor_token"])
# Homogeneous transformation matrix from sensor coordinate frame to ego car frame.
car_from_sensor = transform_matrix(calibrated_sensor['translation'],
Quaternion(
calibrated_sensor['rotation']),
inverse=False)
lidar_pointcloud = LidarPointCloud.from_file(lidar_filepath)
# The lidar pointcloud is defined in the sensor's reference frame.
# We want it in the car's reference frame, so we transform each point
lidar_pointcloud.transform(car_from_sensor)
return lidar_pointcloud.points[:3, :].T
# Creating input and targets
# Let's load the first sample in the train set. We can use that to test the functions
# we'll define next that transform the data to the format we want to input into the model we are training.
sample_token = train_df.first_sample_token.values[56]
# sample_token : cea0bba4b425537cca52b17bf81569a20da1ca6d359f33227f0230d59d9d2881
# 在sample 表里面根据sample_stoken拿到sample
sample = level5data.get("sample", sample_token)
# 拿到上方雷达token 上方雷达数据可视化在 https://www.kaggle.com/tarunpaparaju/lyft-competition-understanding-the-data
sample_lidar_token = sample["data"]["LIDAR_TOP"]
# 在sample_data 表中拿到lidar data , sample_data中也有 picture data
lidar_data = level5data.get("sample_data", sample_lidar_token)
lidar_filepath = level5data.get_sample_data_path(sample_lidar_token)
print(lidar_filepath)
print(os.path.exists(lidar_filepath))
# 在ego_pose 表中拿到 lidar_data 对应的ego_pose
ego_pose = level5data.get("ego_pose", lidar_data["ego_pose_token"])
# 拿到calibrated_sensor 这些token 都可以在https://www.kaggle.com/c/3d-object-detection-for-autonomous-vehicles/discussion/110257#latest-636505的sample_data.json里面找到
calibrated_sensor = level5data.get("calibrated_sensor",
lidar_data["calibrated_sensor_token"])
# 坐标系的转换 从车坐标系转到世界坐标系
# Homogeneous transformation matrix from car frame to world frame.
global_from_car = transform_matrix(ego_pose['translation'],
Quaternion(ego_pose['rotation']),
def main():
# get data
#level5data = LyftDataset(data_path = '../input/', json_path='../input/train_data', verbose=True) # local laptop
level5data = LyftDataset(data_path = '.', json_path='../../input/train_data', verbose=True) # server
classes = ["car", "motorcycle", "bus", "bicycle", "truck", "pedestrian", "other_vehicle", "animal", "emergency_vehicle"]
df = pd.read_csv('host_scenes.csv')
host_count_df = df.groupby("host")['scene_token'].count()
print(host_count_df)
sample_token = df.first_sample_token.values[0]
sample = level5data.get("sample", sample_token)
sample_lidar_token = sample["data"]["LIDAR_TOP"]
lidar_data = level5data.get("sample_data", sample_lidar_token)
lidar_filepath = level5data.get_sample_data_path(sample_lidar_token)
# car and sensor coords
ego_pose = level5data.get("ego_pose", lidar_data["ego_pose_token"])
print("ego_pose: ", ego_pose)
calibrated_sensor = level5data.get("calibrated_sensor", lidar_data["calibrated_sensor_token"])
# Homogeneous transformation matrix from car frame to world frame.
global_from_car = transform_matrix(ego_pose['translation'],
Quaternion(ego_pose['rotation']), inverse=False)
# Homogeneous transformation matrix from sensor coordinate frame to ego car frame
car_from_sensor = transform_matrix(calibrated_sensor['translation'], Quaternion(calibrated_sensor['rotation']),
inverse=False)
lidar_pointcloud = LidarPointCloud.from_file(lidar_filepath)
lidar_pointcloud.transform(car_from_sensor)
# Define hyper parameters
bev_shape = (768, 768, 3)
voxel_size = (0.2,0.2,1.5)
#bev_shape = (2048, 2048, 3)
#voxel_size = (0.1,0.1,1.5)
z_offset = -2.0
box_scale = 0.8
test_coco_targets_for_sample(sample_token=sample_token, output_folder=OUTPUT_ROOT, level5data=level5data,
classes=classes, bev_shape=bev_shape, voxel_size=voxel_size,
z_offset=z_offset, box_scale=box_scale)
train_data_folder = os.path.join(OUTPUT_ROOT, "coco_768")
os.makedirs(train_data_folder, exist_ok=True)
"""
# test on a single scene
prepare_coco_targets_for_scene(df.first_sample_token.values[50],
train_data_folder,
level5data,
classes,
bev_shape, voxel_size, z_offset, box_scale)
test_prepare_coco_targets(df.first_sample_token.values[50],
train_data_folder,
level5data,
bev_shape, voxel_size, z_offset)
"""
# get for all scenes
first_samples = df.first_sample_token.values
for sample in first_samples:
print(sample)
prepare_coco_targets_for_scene(sample,
train_data_folder,
level5data,
classes,
bev_shape, voxel_size, z_offset, box_scale)
print('Mission accomplished!')
def main():
# get data
level5data = LyftDataset(data_path = '.', json_path='../../input/train_data', verbose=True)
classes = ["car", "motorcycle", "bus", "bicycle", "truck", "pedestrian", "other_vehicle", "animal", "emergency_vehicle"]
df = pd.read_csv('host_scenes.csv')
host_count_df = df.groupby("host")['scene_token'].count()
print(host_count_df)
sample_token = df.first_sample_token.values[0]
sample = level5data.get("sample", sample_token)
sample_lidar_token = sample["data"]["LIDAR_TOP"]
lidar_data = level5data.get("sample_data", sample_lidar_token)
lidar_filepath = level5data.get_sample_data_path(sample_lidar_token)
# car and sensor coords
ego_pose = level5data.get("ego_pose", lidar_data["ego_pose_token"])
print("ego_pose: ", ego_pose)
calibrated_sensor = level5data.get("calibrated_sensor", lidar_data["calibrated_sensor_token"])
# Homogeneous transformation matrix from car frame to world frame.
global_from_car = transform_matrix(ego_pose['translation'],
Quaternion(ego_pose['rotation']), inverse=False)
# Homogeneous transformation matrix from sensor coordinate frame to ego car frame.
car_from_sensor = transform_matrix(calibrated_sensor['translation'], Quaternion(calibrated_sensor['rotation']),
inverse=False)
lidar_pointcloud = LidarPointCloud.from_file(lidar_filepath)
# The lidar pointcloud is defined in the sensor's reference frame.
# We want it in the car's reference frame, so we transform each point
lidar_pointcloud.transform(car_from_sensor)
# Define hyper parameters
#bev_shape = (768, 768, 3)
#voxel_size = (0.2,0.2,1.5)
bev_shape = (1024, 1024, 3)
voxel_size = (0.2,0.2,1.5)
z_offset = -2.0
box_scale = 0.8
#map_mask = level5data.map[0]["mask"]
#print(map_mask)
#plot_semantic_map(map_mask, ego_pose)
#sample_token = train_df.first_sample_token.values[10]
#bev = plot_bev_and_boxes(sample_token, level5data)
train_data_folder = os.path.join(OUTPUT_ROOT, "bev_data_1024")
validation_data_folder = os.path.join(OUTPUT_ROOT, "bev_data_1024")
os.makedirs(train_data_folder, exist_ok=True)
os.makedirs(validation_data_folder, exist_ok=True)
# test on a single scene
prepare_training_data_for_scene(df.first_sample_token.values[50],
train_data_folder,
level5data,
classes,
bev_shape, voxel_size, z_offset, box_scale)
test_prepare_training_data(df.first_sample_token.values[50], train_data_folder)
# get for all scenes
first_samples = df.first_sample_token.values
print(first_samples)
for sample in first_samples:
print(sample)
prepare_training_data_for_scene(sample,
train_data_folder,
level5data,
classes,
bev_shape, voxel_size, z_offset, box_scale)
# multiprocessing option
#for df, data_folder in [(train_df, train_data_folder), (validation_df, validation_data_folder)]:
# prepare_data_pool(df, data_folder, bev_shape, voxel_size, z_offset, box_scale)
print('Mission accomplished!')
class LyftLevel5Dataset(data.Dataset):
def __init__(self, phase):
df_name = phase if phase is not 'test' else 'sample_submission'
dataset = 'train' if phase is not 'test' else 'test'
self.phase = phase
self.df = pd.read_csv(
os.path.join(cfg.work_dir, 'data', df_name + '.csv'))
self.lyft_dataset = LyftDataset(
data_path=os.path.join(cfg.input_dir, dataset),
json_path=os.path.join(cfg.input_dir, dataset, 'data'),
verbose=False)
def voxelize_pointclouds(self, pointclouds):
# Shuffle the pointclouds.
np.random.shuffle(pointclouds)
# x, y and z correspond to vox_width, vox_height and vox_depth, respectively.
voxel_coords = (
(pointclouds[:, :3] -
np.array([cfg.xrange[0], cfg.yrange[0], cfg.zrange[0]])) /
(cfg.vox_width, cfg.vox_height, cfg.vox_depth)).astype(np.int32)
# Convert to (z, y, x) to sample unique voxel.
voxel_coords = voxel_coords[:, [2, 1, 0]]
voxel_coords, inv_ind, voxel_counts = np.unique(voxel_coords,
axis=0,
return_inverse=True,
return_counts=True)
# Fill each voxel with voxel feature.
voxel_features = []
for i in range(len(voxel_coords)):
voxel = np.zeros((cfg.pointclouds_per_vox, 7), dtype=np.float32)
pts = pointclouds[inv_ind == i]
if voxel_counts[i] > cfg.pointclouds_per_vox:
pts = pts[:cfg.pointclouds_per_vox, :]
voxel_counts[i] = cfg.pointclouds_per_vox
# Augment the points: (x, y, z) -> (x, y, z, i, x-mean, y-mean, z-mean).
voxel[:pts.shape[0], :] = np.concatenate(
(pts, pts[:, :3] - np.mean(pts[:, :3], 0)), axis=1)
voxel_features.append(voxel)
return np.array(voxel_features), voxel_coords
def cal_target(self, gt_boxes3d):
anchors_d = np.sqrt(cfg.anchors[:, 3]**2 + cfg.anchors[:, 4]**2)
pos_equal_one = np.zeros((*cfg.feature_map_shape, cfg.ac_rot_z))
neg_equal_one = np.zeros((*cfg.feature_map_shape, cfg.ac_rot_z))
targets = np.zeros((*cfg.feature_map_shape, 7 * cfg.ac_rot_z))
gt_boxes3d_xyzlwhr = np.array([[
gt_box3d.center[0], gt_box3d.center[1], gt_box3d.center[2],
gt_box3d.wlh[1], gt_box3d.wlh[0], gt_box3d.wlh[2],
gt_box3d.orientation.yaw_pitch_roll[0]
] for gt_box3d in gt_boxes3d])
# Some class is not included in dataset.
try:
# Only taken into account rotation around yaw axis.
# It means bottom-corners equal to top-corner.
gt_boxes2d_corners = utils.boxes3d_to_corners(gt_boxes3d_xyzlwhr)
except Exception as e:
return pos_equal_one, neg_equal_one, targets, False
ac_boxes2d_corners = utils.boxes3d_to_corners(cfg.anchors)
ac_boxes2d = utils.boxes2d_four_corners_to_two_corners(
ac_boxes2d_corners)
gt_boxes2d = utils.boxes2d_four_corners_to_two_corners(
gt_boxes2d_corners)
# iou: [num_ac_boxes2d, num_gt_boxes2d]
iou = bbox_overlaps(
np.ascontiguousarray(ac_boxes2d).astype(np.float32),
np.ascontiguousarray(gt_boxes2d).astype(np.float32))
# id_highest: [num_gt_boxes2d]
# - For each gt_boxes2d, index of max iou-scored anchor box.
id_highest = np.argmax(iou.T, axis=1)
# id_highest_gt: [num_gt_boxes2d]
# - Ranged from 0 to num_gt_boxes2d - 1.
id_highest_gt = np.arange(iou.T.shape[0])
# For gt_boxes3d, mask stands for filter of box with highest anchor which has iou > 0.
mask = iou.T[id_highest_gt, id_highest] > 0
# Get rid of those gt_boxes3d with iou of 0 with each anchor.
id_highest, id_highest_gt = id_highest[mask], id_highest_gt[mask]
# In the table of iou, every (ac_boxes2d and gt_boxes2d) pair with iou > iou_threshold_p.
# id_pos: [num_pos_ac_boxes2d]
# id_pos_gt: [num_pos_ac_boxes2d]
id_pos, id_pos_gt = np.where(iou > cfg.iou_pos_threshold)
# In the table of iou, every anchor with iou < iou_threshold_n with all gt_boxes2d.
id_neg = np.where(
np.sum(iou < cfg.iou_neg_threshold, axis=1) == iou.shape[1])[0]
id_pos = np.concatenate([id_pos, id_highest])
id_pos_gt = np.concatenate([id_pos_gt, id_highest_gt])
id_pos, index = np.unique(id_pos, return_index=True)
# The index of id_pos of anchor appears first time.
id_pos_gt = id_pos_gt[index]
id_neg.sort()
# Cal the target and set the equal one.
index_x, index_y, index_z = np.unravel_index(
id_pos, (*cfg.feature_map_shape, cfg.ac_rot_z))
pos_equal_one[index_x, index_y, index_z] = 1
targets[index_x, index_y, np.array(index_z) * 7] = \
(gt_boxes3d_xyzlwhr[id_pos_gt, 0] - cfg.anchors[id_pos, 0]) / anchors_d[id_pos]
targets[index_x, index_y, np.array(index_z) * 7 + 1] = \
(gt_boxes3d_xyzlwhr[id_pos_gt, 1] - cfg.anchors[id_pos, 1]) / anchors_d[id_pos]
targets[index_x, index_y, np.array(index_z) * 7 + 2] = \
(gt_boxes3d_xyzlwhr[id_pos_gt, 2] - cfg.anchors[id_pos, 2]) / cfg.anchors[id_pos, 3]
targets[index_x, index_y, np.array(index_z) * 7 + 3] = \
np.log(gt_boxes3d_xyzlwhr[id_pos_gt, 3] / cfg.anchors[id_pos, 3])
targets[index_x, index_y, np.array(index_z) * 7 + 4] = \
np.log(gt_boxes3d_xyzlwhr[id_pos_gt, 4] / cfg.anchors[id_pos, 4])
targets[index_x, index_y, np.array(index_z) * 7 + 5] = \
np.log(gt_boxes3d_xyzlwhr[id_pos_gt, 5] / cfg.anchors[id_pos, 5])
targets[index_x, index_y, np.array(index_z) * 7 + 6] = \
np.sin(gt_boxes3d_xyzlwhr[id_pos_gt, 6] - cfg.anchors[id_pos, 6])
index_x, index_y, index_z = np.unravel_index(
id_neg, (*cfg.feature_map_shape, cfg.ac_rot_z))
neg_equal_one[index_x, index_y, index_z] = 1
# To avoid a box be pos/neg in the same time
index_x, index_y, index_z = np.unravel_index(
id_highest, (*cfg.feature_map_shape, cfg.ac_rot_z))
neg_equal_one[index_x, index_y, index_z] = 0
return pos_equal_one, neg_equal_one, targets, True
def __getitem__(self, idx):
# Point clouds of lidar are positioned at the sensor frame,
# while gt_boxes3d at the global frame.
sample_token = 'Id' if self.phase is 'test' else 'sample_token'
sample = self.lyft_dataset.get('sample', self.df[sample_token][idx])
lidar_info = self.lyft_dataset.get('sample_data',
sample['data']['LIDAR_TOP'])
lidar_data_path = self.lyft_dataset.get_sample_data_path(
sample['data']['LIDAR_TOP'])
gt_boxes3d = self.lyft_dataset.get_boxes(sample['data']['LIDAR_TOP'])
ego_pose = self.lyft_dataset.get('ego_pose',
lidar_info['ego_pose_token'])
calibrated_sensor = self.lyft_dataset.get(
'calibrated_sensor', lidar_info['calibrated_sensor_token'])
# pointclouds w.r.t. the sensor frame: [4 xyzi, num_points]
pointclouds = LidarPointCloud.from_file(lidar_data_path)
# pointclouds: [4 xyzi, num_points] -> [num_points, 4 xyzi]
pointclouds = pointclouds.points.transpose(1, 0)
if self.phase is not 'test':
# Convert gt_boxes3d from the global frame to the sensor frame.
gt_boxes3d = utils.convert_boxes3d_from_global_to_sensor_frame(
gt_boxes3d, ego_pose, calibrated_sensor)
# Data augmentation.
#pointclouds, gt_box3d = aug_data(pointclouds, gt_box3d)
# Filter point clouds and gt_boxes3d within a specific range and class name.
pointclouds, gt_boxes3d = utils.filter_pointclouds_gt_boxes3d(
pointclouds, gt_boxes3d, cfg.class_name)
# Voxelize point clouds.
voxel_features, voxel_coords = self.voxelize_pointclouds(
pointclouds)
# Encode bounding boxes.
pos_equal_one, neg_equal_one, targets, exception = self.cal_target(
gt_boxes3d)
return voxel_features, voxel_coords, pos_equal_one, neg_equal_one, targets, exception
else:
pointclouds = utils.filter_pointclouds_gt_boxes3d(pointclouds)
# Voxelize point clouds.
voxel_features, voxel_coords = self.voxelize_pointclouds(
pointclouds)
return voxel_features, voxel_coords, self.df[sample_token][
idx], ego_pose, calibrated_sensor
def __len__(self):
return len(self.df)
verbose=True)
winName = 'point'
winName2 = 'camera'
cv.namedWindow(winName, cv.WINDOW_NORMAL)
cv.namedWindow(winName2, cv.WINDOW_NORMAL)
#level5data.list_scenes() #get all of the sence
my_scene = level5data.scene[0] #chosse [0]for analysing
my_sample_token = my_scene["first_sample_token"] #choose token [0] for analyse
my_sample = level5data.get('sample', my_sample_token)
KEY = 200
while 1:
if KEY == 13:
break
img_token = level5data.get('sample_data', my_sample['data']['CAM_FRONT'])
img_filepath = level5data.get_sample_data_path(
my_sample['data']['CAM_FRONT'])
cap = cv.VideoCapture(str(img_filepath))
lidar_filepath = level5data.get_sample_data_path(
my_sample['data']['LIDAR_TOP'])
lidar_pointcloud = LidarPointCloud.from_file(lidar_filepath)
hasFrame, frame = cap.read() #显示图像读取状态并获取图像帧
# network
blob = cv.dnn.blobFromImage(frame,
1 / 255, (inpWidth, inpHeight), [0, 0, 0],
1,
crop=False)
net.setInput(blob)
outs = net.forward(getOutputsNames(net))
# 后期处理
class Lyft(SequenceDataset):
def __init__(self, root, split=None):
self.data = LyftDataset(data_path=root,
json_path=os.path.join(
root,
os.path.basename(os.path.normpath(root))),
verbose=False)
self.explorer = LyftDatasetExplorer(self.data)
super().__init__(root, split)
def _get_sequences(self):
return [(scene["first_sample_token"], "CAM_FRONT")
for scene in self.data.scene]
def _get_samples(self, sequence):
FIRST_TOKEN, CAMERA = sequence
next_token = FIRST_TOKEN
samples = []
while next_token != "":
sample = self.data.get("sample", next_token)
next_token = sample["next"]
image = sample["data"][CAMERA]
samples.append(image)
return samples
def _load_sample(self, seq_i, sample):
sd_record = self.data.get("sample_data", sample)
cs_record = self.data.get("calibrated_sensor",
sd_record["calibrated_sensor_token"])
pose_record = self.data.get("ego_pose", sd_record["ego_pose_token"])
K = np.array(cs_record["camera_intrinsic"]).astype(np.float32)
T = Quaternion(pose_record["rotation"]).transformation_matrix
T[:3, 3] = np.array(pose_record["translation"])
#print("before", T)
T = T @ np.array([
[0, 0, 1, 0],
[-1, 0, 0, 0],
[0, -1, 0, 0],
[0, 0, 0, 1],
])
#print("after", T)
img, K = self._load_image_from_disk(
self.data.get_sample_data_path(sample), K)
return [img, T, K]
def _load_depth(self, sample):
sample_data = self.data.get("sample_data", sample)
LIDAR_TOKEN = self.data.get(
"sample", sample_data["sample_token"])["data"]["LIDAR_TOP"]
points, depths, image = self.explorer.map_pointcloud_to_image(
LIDAR_TOKEN, sample)
W, H = image.size
crop, scale = self.calc_crop(H, W)
dmap = np.zeros(self.imHW, dtype=np.float64)
coords = points[:2, :] * scale
coords[1] -= crop
mask = np.logical_and(coords[1, :] >= 0, coords[1, :] < self.imHW[0])
coords = coords[:, mask].astype(np.int)
depths = depths[mask]
dmap[coords[1], coords[0]] = depths
return dmap
def extract_boxed_clouds(num_entries, lyftd: LyftDataset, point_threshold=1024,
while_list_type_str=['car', 'pedestrian', 'bicycle'],
save_file=os.path.join(ARTIFACT_PATH, "val_pc.pickle")):
point_clouds_list = []
one_hot_vector_list = []
train_objects = parse_train_csv()
# get train box information, center and wlh
for idx in range(train_objects.shape[0]):
box, sample_data_token = extract_single_box(train_objects, idx=idx)
print(box.name)
if box.name not in while_list_type_str:
continue
else:
type_index = while_list_type_str.index(box.name)
one_hot_vector = np.zeros(3, dtype=np.bool)
one_hot_vector[type_index] = True
lidar_file_path = lyftd.get_sample_data_path(sample_data_token)
lpc = LidarPointCloud.from_file(lidar_file_path)
# get the point cloud associated with this cloud
# mask out the point clouds
mask = points_in_box(box, lpc.points[0:3, :])
masked_ldp_points = lpc.points[:, mask]
# transform the masked point clouds to frustum coordinates
# For the time being, just translate it the center coordinates
# for k in range(masked_ldp_points.shape[1]):
# masked_ldp_points[0:3, k] -= box.center
# Show number of points
print("number of cloud points: {}".format(masked_ldp_points.shape[1]))
##Patch: calibration using KITTI calibration data
ncpc = calib_point_cloud(masked_ldp_points[0:3, :].T)
masked_ldp_points[0:3, :] = np.transpose(ncpc)
rescale_lidar_intensity(masked_ldp_points, 0.2)
if masked_ldp_points.shape[1] > point_threshold:
# Store the results
point_clouds_list.append(masked_ldp_points)
one_hot_vector_list.append(one_hot_vector)
if len(point_clouds_list) >= num_entries:
break
point_clouds_list = rearrange_point_clouds(point_clouds_list)
one_hot_vector_list = rearrange_one_hot_vector(one_hot_vector_list)
# save the file
with open(save_file, 'wb') as fp:
save_dict = {"pcl": point_clouds_list,
"ohv": one_hot_vector_list}
pickle.dump(save_dict, fp)
return point_clouds_list, one_hot_vector_list
token_dirs = [cfg.DATA.TOKEN_TRAIN_DIR, cfg.DATA.TOKEN_VAL_DIR]
# loop through the training and validation data frames
for df, box_dir, lidar_dir, token_dir in zip(dfs, box_dirs, lidar_dirs,
token_dirs):
data_dict = {}
token_list = []
# loop through each sample in a scene
for token in tqdm(df.first_sample_token):
while token:
sample = l5d.get('sample', token)
lidar_token = sample['data']['LIDAR_TOP']
try:
# there may be currupted lidar files - mark these tokens
# by setting the file path to the lidar file as None
lidar_filepath = l5d.get_sample_data_path(lidar_token)
lidar_pointcloud = LidarPointCloud.from_file(lidar_filepath)
except Exception as e:
print('Failed to load LIDAR cloud for {}: {}:'.format(
token, e))
prev_token = sample['prev']
data_dict[token] = {
'lidar_fp': None,
'ego_pose': None,
'cal_sensor': None,
'boxes': None,
'prev_token': prev_token
}
token = sample['next']
continue
# get the lidar, ego and sensor objects for the token
