def evaluation(self, detections, output_dir): """ detection When you want to eval your own dataset, you MUST set correct the z axis and box z center. If you want to eval by my KITTI eval function, you must provide the correct format annotations. ground_truth_annotations format: { bbox: [N, 4], if you fill fake data, MUST HAVE >25 HEIGHT!!!!!! alpha: [N], you can use -10 to ignore it. occluded: [N], you can use zero. truncated: [N], you can use zero. name: [N] location: [N, 3] center of 3d box. dimensions: [N, 3] dim of 3d box. rotation_y: [N] angle. } all fields must be filled, but some fields can fill zero. """ if "annos" not in self._kitti_infos[0]: return None gt_annos = [info["annos"] for info in self._kitti_infos] dt_annos = self.convert_detection_to_kitti_annos(detections) # firstly convert standard detection to kitti-format dt annos z_axis = 1 # KITTI camera format use y as regular "z" axis. z_center = 1.0 # KITTI camera box's center is [0.5, 1, 0.5] # for regular raw lidar data, z_axis = 2, z_center = 0.5. result_official_dict = get_official_eval_result( gt_annos, dt_annos, self._class_names, z_axis=z_axis, z_center=z_center) result_coco = get_coco_eval_result( gt_annos, dt_annos, self._class_names, z_axis=z_axis, z_center=z_center) return { "results": { "official": result_official_dict["result"], "coco": result_coco["result"], }, "detail": { "eval.kitti": { "official": result_official_dict["detail"], "coco": result_coco["detail"] } }, }
def evaluation(self, dt_annos): """dt_annos have same format as ground_truth_annotations. When you want to eval your own dataset, you MUST set correct the z axis and box z center. """ gt_annos = self.ground_truth_annotations if gt_annos is None: return None, None z_axis = 1 # KITTI camera format use y as regular "z" axis. z_center = 1.0 # KITTI camera box's center is [0.5, 1, 0.5] # for regular raw lidar data, z_axis = 2, z_center = 0.5. result_official = get_official_eval_result(gt_annos, dt_annos, self._class_names, z_axis=z_axis, z_center=z_center) result_coco = get_coco_eval_result(gt_annos, dt_annos, self._class_names, z_axis=z_axis, z_center=z_center) return result_official, result_coco
def evaluation(self, detections, output_dir): """ detection When you want to eval your own dataset, you MUST set correct the z axis and box z center. """ gt_annos = self.ground_truth_annotations if gt_annos is None: return None dt_annos = self.convert_detection_to_kitti_annos(detections) # firstly convert standard detection to kitti-format dt annos z_axis = 1 # KITTI camera format use y as regular "z" axis. z_center = 1.0 # KITTI camera box's center is [0.5, 1, 0.5] # for regular raw lidar data, z_axis = 2, z_center = 0.5. result_official_dict = get_official_eval_result(gt_annos, dt_annos, self._class_names, z_axis=z_axis, z_center=z_center) result_coco = get_coco_eval_result(gt_annos, dt_annos, self._class_names, z_axis=z_axis, z_center=z_center) return { "results": { "official": result_official_dict["result"], "coco": result_coco["result"], }, "detail": { "eval.kitti": { "official": result_official_dict["detail"], "coco": result_coco["detail"] } }, }
def evaluation_kitti(self, detections, output_dir): """eval by kitti evaluation tool. I use num_lidar_pts to set easy, mod, hard. easy: num>15, mod: num>7, hard: num>0. """ print("++++++++NuScenes KITTI unofficial Evaluation:") print( "++++++++easy: num_lidar_pts>15, mod: num_lidar_pts>7, hard: num_lidar_pts>0" ) print("++++++++The bbox AP is invalid. Don't forget to ignore it.") class_names = self._class_names gt_annos = self.ground_truth_annotations if gt_annos is None: return None gt_annos = deepcopy(gt_annos) detections = deepcopy(detections) dt_annos = [] for det in detections: final_box_preds = det["box3d_lidar"].detach().cpu().numpy() label_preds = det["label_preds"].detach().cpu().numpy() scores = det["scores"].detach().cpu().numpy() anno = kitti.get_start_result_anno() num_example = 0 box3d_lidar = final_box_preds for j in range(box3d_lidar.shape[0]): anno["bbox"].append(np.array([0, 0, 50, 50])) anno["alpha"].append(-10) anno["dimensions"].append(box3d_lidar[j, 3:6]) anno["location"].append(box3d_lidar[j, :3]) anno["rotation_y"].append(box3d_lidar[j, 6]) anno["name"].append(class_names[int(label_preds[j])]) anno["truncated"].append(0.0) anno["occluded"].append(0) anno["score"].append(scores[j]) num_example += 1 if num_example != 0: anno = {n: np.stack(v) for n, v in anno.items()} dt_annos.append(anno) else: dt_annos.append(kitti.empty_result_anno()) num_example = dt_annos[-1]["name"].shape[0] dt_annos[-1]["metadata"] = det["metadata"] for anno in gt_annos: names = anno["name"].tolist() mapped_names = [] for n in names: if n in self.NameMapping: mapped_names.append(self.NameMapping[n]) else: mapped_names.append(n) anno["name"] = np.array(mapped_names) for anno in dt_annos: names = anno["name"].tolist() mapped_names = [] for n in names: if n in self.NameMapping: mapped_names.append(self.NameMapping[n]) else: mapped_names.append(n) anno["name"] = np.array(mapped_names) mapped_class_names = [] for n in self._class_names: if n in self.NameMapping: mapped_class_names.append(self.NameMapping[n]) else: mapped_class_names.append(n) z_axis = 2 z_center = 0.5 # for regular raw lidar data, z_axis = 2, z_center = 0.5. result_official_dict = get_official_eval_result(gt_annos, dt_annos, mapped_class_names, z_axis=z_axis, z_center=z_center) result_coco = get_coco_eval_result(gt_annos, dt_annos, mapped_class_names, z_axis=z_axis, z_center=z_center) return { "results": { "official": result_official_dict["result"], "coco": result_coco["result"], }, "detail": { "official": result_official_dict["detail"], "coco": result_coco["detail"], }, }
def train(config_path, model_dir, result_path=None, create_folder=False, display_step=50, summary_step=5, pickle_result=True): """train a VoxelNet model specified by a config file. """ if create_folder: if pathlib.Path(model_dir).exists(): model_dir = torchplus.train.create_folder(model_dir) model_dir = pathlib.Path(model_dir) model_dir.mkdir(parents=True, exist_ok=True) eval_checkpoint_dir = model_dir / 'eval_checkpoints' eval_checkpoint_dir.mkdir(parents=True, exist_ok=True) if result_path is None: result_path = model_dir / 'results' config_file_bkp = "pipeline.config" config = pipeline_pb2.TrainEvalPipelineConfig() with open(config_path, "r") as f: proto_str = f.read() text_format.Merge(proto_str, config) shutil.copyfile(config_path, str(model_dir / config_file_bkp)) input_cfg = config.train_input_reader eval_input_cfg = config.eval_input_reader model_cfg = config.model.second train_cfg = config.train_config class_names = list(input_cfg.class_names) ###################### # BUILD VOXEL GENERATOR ###################### voxel_generator = voxel_builder.build(model_cfg.voxel_generator) ###################### # BUILD TARGET ASSIGNER ###################### bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]] box_coder = box_coder_builder.build(model_cfg.box_coder) target_assigner_cfg = model_cfg.target_assigner target_assigner = target_assigner_builder.build(target_assigner_cfg, bv_range, box_coder) ###################### # BUILD NET ###################### center_limit_range = model_cfg.post_center_limit_range net = second_builder.build(model_cfg, voxel_generator, target_assigner) net.cuda() # net_train = torch.nn.DataParallel(net).cuda() print("num_trainable parameters:", len(list(net.parameters()))) # for n, p in net.named_parameters(): # print(n, p.shape) ###################### # BUILD OPTIMIZER ###################### # we need global_step to create lr_scheduler, so restore net first. torchplus.train.try_restore_latest_checkpoints(model_dir, [net]) gstep = net.get_global_step() - 1 optimizer_cfg = train_cfg.optimizer if train_cfg.enable_mixed_precision: net.half() net.metrics_to_float() net.convert_norm_to_float(net) optimizer = optimizer_builder.build(optimizer_cfg, net.parameters()) if train_cfg.enable_mixed_precision: loss_scale = train_cfg.loss_scale_factor mixed_optimizer = torchplus.train.MixedPrecisionWrapper( optimizer, loss_scale) else: mixed_optimizer = optimizer # must restore optimizer AFTER using MixedPrecisionWrapper torchplus.train.try_restore_latest_checkpoints(model_dir, [mixed_optimizer]) lr_scheduler = lr_scheduler_builder.build(optimizer_cfg, optimizer, gstep) if train_cfg.enable_mixed_precision: float_dtype = torch.float16 else: float_dtype = torch.float32 ###################### # PREPARE INPUT ###################### dataset = input_reader_builder.build(input_cfg, model_cfg, training=True, voxel_generator=voxel_generator, target_assigner=target_assigner) eval_dataset = input_reader_builder.build(eval_input_cfg, model_cfg, training=False, voxel_generator=voxel_generator, target_assigner=target_assigner) def _worker_init_fn(worker_id): time_seed = np.array(time.time(), dtype=np.int32) np.random.seed(time_seed + worker_id) print(f"WORKER {worker_id} seed:", np.random.get_state()[1][0]) dataloader = torch.utils.data.DataLoader(dataset, batch_size=input_cfg.batch_size, shuffle=True, num_workers=input_cfg.num_workers, pin_memory=False, collate_fn=merge_second_batch, worker_init_fn=_worker_init_fn) eval_dataloader = torch.utils.data.DataLoader( eval_dataset, batch_size=eval_input_cfg.batch_size, shuffle=False, num_workers=eval_input_cfg.num_workers, pin_memory=False, collate_fn=merge_second_batch) data_iter = iter(dataloader) ###################### # TRAINING ###################### log_path = model_dir / 'log.txt' logf = open(log_path, 'a') logf.write(proto_str) logf.write("\n") summary_dir = model_dir / 'summary' summary_dir.mkdir(parents=True, exist_ok=True) writer = SummaryWriter(str(summary_dir)) total_step_elapsed = 0 remain_steps = train_cfg.steps - net.get_global_step() t = time.time() ckpt_start_time = t total_loop = train_cfg.steps // train_cfg.steps_per_eval + 1 # total_loop = remain_steps // train_cfg.steps_per_eval + 1 clear_metrics_every_epoch = train_cfg.clear_metrics_every_epoch if train_cfg.steps % train_cfg.steps_per_eval == 0: total_loop -= 1 mixed_optimizer.zero_grad() try: for _ in range(total_loop): if total_step_elapsed + train_cfg.steps_per_eval > train_cfg.steps: steps = train_cfg.steps % train_cfg.steps_per_eval else: steps = train_cfg.steps_per_eval for step in range(steps): lr_scheduler.step() try: example = next(data_iter) except StopIteration: print("end epoch") if clear_metrics_every_epoch: net.clear_metrics() data_iter = iter(dataloader) example = next(data_iter) example_torch = example_convert_to_torch(example, float_dtype) batch_size = example["anchors"].shape[0] ret_dict = net(example_torch) # box_preds = ret_dict["box_preds"] cls_preds = ret_dict["cls_preds"] loss = ret_dict["loss"].mean() cls_loss_reduced = ret_dict["cls_loss_reduced"].mean() loc_loss_reduced = ret_dict["loc_loss_reduced"].mean() cls_pos_loss = ret_dict["cls_pos_loss"] cls_neg_loss = ret_dict["cls_neg_loss"] loc_loss = ret_dict["loc_loss"] cls_loss = ret_dict["cls_loss"] dir_loss_reduced = ret_dict["dir_loss_reduced"] cared = ret_dict["cared"] labels = example_torch["labels"] if train_cfg.enable_mixed_precision: loss *= loss_scale loss.backward() torch.nn.utils.clip_grad_norm_(net.parameters(), 10.0) mixed_optimizer.step() mixed_optimizer.zero_grad() net.update_global_step() net_metrics = net.update_metrics(cls_loss_reduced, loc_loss_reduced, cls_preds, labels, cared) step_time = (time.time() - t) t = time.time() metrics = {} num_pos = int((labels > 0)[0].float().sum().cpu().numpy()) num_neg = int((labels == 0)[0].float().sum().cpu().numpy()) if 'anchors_mask' not in example_torch: num_anchors = example_torch['anchors'].shape[1] else: num_anchors = int(example_torch['anchors_mask'][0].sum()) global_step = net.get_global_step() if global_step % display_step == 0: loc_loss_elem = [ float(loc_loss[:, :, i].sum().detach().cpu().numpy() / batch_size) for i in range(loc_loss.shape[-1]) ] metrics["step"] = global_step metrics["steptime"] = step_time metrics.update(net_metrics) metrics["loss"] = {} metrics["loss"]["loc_elem"] = loc_loss_elem metrics["loss"]["cls_pos_rt"] = float( cls_pos_loss.detach().cpu().numpy()) metrics["loss"]["cls_neg_rt"] = float( cls_neg_loss.detach().cpu().numpy()) # if unlabeled_training: # metrics["loss"]["diff_rt"] = float( # diff_loc_loss_reduced.detach().cpu().numpy()) if model_cfg.use_direction_classifier: metrics["loss"]["dir_rt"] = float( dir_loss_reduced.detach().cpu().numpy()) metrics["num_vox"] = int(example_torch["voxels"].shape[0]) metrics["num_pos"] = int(num_pos) metrics["num_neg"] = int(num_neg) metrics["num_anchors"] = int(num_anchors) metrics["lr"] = float( mixed_optimizer.param_groups[0]['lr']) metrics["image_idx"] = example['image_idx'][0] flatted_metrics = flat_nested_json_dict(metrics) flatted_summarys = flat_nested_json_dict(metrics, "/") for k, v in flatted_summarys.items(): if isinstance(v, (list, tuple)): v = {str(i): e for i, e in enumerate(v)} writer.add_scalars(k, v, global_step) else: writer.add_scalar(k, v, global_step) metrics_str_list = [] for k, v in flatted_metrics.items(): if isinstance(v, float): metrics_str_list.append(f"{k}={v:.3}") elif isinstance(v, (list, tuple)): if v and isinstance(v[0], float): v_str = ', '.join([f"{e:.3}" for e in v]) metrics_str_list.append(f"{k}=[{v_str}]") else: metrics_str_list.append(f"{k}={v}") else: metrics_str_list.append(f"{k}={v}") log_str = ', '.join(metrics_str_list) print(log_str, file=logf) print(log_str) ckpt_elasped_time = time.time() - ckpt_start_time if ckpt_elasped_time > train_cfg.save_checkpoints_secs: torchplus.train.save_models(model_dir, [net, optimizer], net.get_global_step()) ckpt_start_time = time.time() total_step_elapsed += steps torchplus.train.save_models(model_dir, [net, optimizer], net.get_global_step()) # Ensure that all evaluation points are saved forever torchplus.train.save_models(eval_checkpoint_dir, [net, optimizer], net.get_global_step(), max_to_keep=100) net.eval() result_path_step = result_path / f"step_{net.get_global_step()}" result_path_step.mkdir(parents=True, exist_ok=True) print("#################################") print("#################################", file=logf) print("# EVAL") print("# EVAL", file=logf) print("#################################") print("#################################", file=logf) print("Generate output labels...") print("Generate output labels...", file=logf) t = time.time() dt_annos = [] prog_bar = ProgressBar() prog_bar.start(len(eval_dataset) // eval_input_cfg.batch_size + 1) for example in iter(eval_dataloader): example = example_convert_to_torch(example, float_dtype) if pickle_result: dt_annos += predict_kitti_to_anno(net, example, class_names, center_limit_range, model_cfg.lidar_input) else: _predict_kitti_to_file(net, example, result_path_step, class_names, center_limit_range, model_cfg.lidar_input) prog_bar.print_bar() sec_per_ex = len(eval_dataset) / (time.time() - t) print(f"avg forward time per example: {net.avg_forward_time:.3f}") print( f"avg postprocess time per example: {net.avg_postprocess_time:.3f}" ) net.clear_time_metrics() print(f'generate label finished({sec_per_ex:.2f}/s). start eval:') print(f'generate label finished({sec_per_ex:.2f}/s). start eval:', file=logf) gt_annos = [ info["annos"] for info in eval_dataset.dataset.kitti_infos ] if not pickle_result: dt_annos = kitti.get_label_annos(result_path_step) result, mAPbbox, mAPbev, mAP3d, mAPaos = get_official_eval_result( gt_annos, dt_annos, class_names, return_data=True) print(result, file=logf) print(result) writer.add_text('eval_result', result, global_step) for i, class_name in enumerate(class_names): writer.add_scalar('bev_ap:{}'.format(class_name), mAPbev[i, 1, 0], global_step) writer.add_scalar('3d_ap:{}'.format(class_name), mAP3d[i, 1, 0], global_step) writer.add_scalar('aos_ap:{}'.format(class_name), mAPaos[i, 1, 0], global_step) writer.add_scalar('bev_map', np.mean(mAPbev[:, 1, 0]), global_step) writer.add_scalar('3d_map', np.mean(mAP3d[:, 1, 0]), global_step) writer.add_scalar('aos_map', np.mean(mAPaos[:, 1, 0]), global_step) result = get_coco_eval_result(gt_annos, dt_annos, class_names) print(result, file=logf) print(result) if pickle_result: with open(result_path_step / "result.pkl", 'wb') as f: pickle.dump(dt_annos, f) writer.add_text('eval_result', result, global_step) net.train() except Exception as e: torchplus.train.save_models(model_dir, [net, optimizer], net.get_global_step()) logf.close() raise e # save model before exit torchplus.train.save_models(model_dir, [net, optimizer], net.get_global_step()) logf.close()
def evaluate(config_path, model_dir, result_path=None, predict_test=False, ckpt_path=None, ref_detfile=None, pickle_result=True): model_dir = pathlib.Path(model_dir) if predict_test: result_name = 'predict_test' else: result_name = 'eval_results' if result_path is None: result_path = model_dir / result_name else: result_path = pathlib.Path(result_path) config = pipeline_pb2.TrainEvalPipelineConfig() with open(config_path, "r") as f: proto_str = f.read() text_format.Merge(proto_str, config) input_cfg = config.eval_input_reader model_cfg = config.model.second train_cfg = config.train_config class_names = list(input_cfg.class_names) center_limit_range = model_cfg.post_center_limit_range ###################### # BUILD VOXEL GENERATOR ###################### voxel_generator = voxel_builder.build(model_cfg.voxel_generator) bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]] box_coder = box_coder_builder.build(model_cfg.box_coder) target_assigner_cfg = model_cfg.target_assigner target_assigner = target_assigner_builder.build(target_assigner_cfg, bv_range, box_coder) net = second_builder.build(model_cfg, voxel_generator, target_assigner) net.cuda() if train_cfg.enable_mixed_precision: net.half() net.metrics_to_float() net.convert_norm_to_float(net) if ckpt_path is None: torchplus.train.try_restore_latest_checkpoints(model_dir, [net]) else: torchplus.train.restore(ckpt_path, net) eval_dataset = input_reader_builder.build(input_cfg, model_cfg, training=False, voxel_generator=voxel_generator, target_assigner=target_assigner) eval_dataloader = torch.utils.data.DataLoader( eval_dataset, batch_size=input_cfg.batch_size, shuffle=False, num_workers=input_cfg.num_workers, pin_memory=False, collate_fn=merge_second_batch) if train_cfg.enable_mixed_precision: float_dtype = torch.float16 else: float_dtype = torch.float32 net.eval() result_path_step = result_path / f"step_{net.get_global_step()}" result_path_step.mkdir(parents=True, exist_ok=True) t = time.time() dt_annos = [] global_set = None print("Generate output labels...") bar = ProgressBar() bar.start(len(eval_dataset) // input_cfg.batch_size + 1) for example in iter(eval_dataloader): example = example_convert_to_torch(example, float_dtype) if pickle_result: dt_annos += predict_kitti_to_anno(net, example, class_names, center_limit_range, model_cfg.lidar_input, global_set) else: _predict_kitti_to_file(net, example, result_path_step, class_names, center_limit_range, model_cfg.lidar_input) bar.print_bar() sec_per_example = len(eval_dataset) / (time.time() - t) print(f'generate label finished({sec_per_example:.2f}/s). start eval:') print(f"avg forward time per example: {net.avg_forward_time:.3f}") print(f"avg postprocess time per example: {net.avg_postprocess_time:.3f}") if not predict_test: gt_annos = [info["annos"] for info in eval_dataset.dataset.kitti_infos] if not pickle_result: dt_annos = kitti.get_label_annos(result_path_step) result = get_official_eval_result(gt_annos, dt_annos, class_names) print(result) result = get_coco_eval_result(gt_annos, dt_annos, class_names) print(result) if pickle_result: with open(result_path_step / "result.pkl", 'wb') as f: pickle.dump(dt_annos, f)
def evaluate(config_path, model_dir, use_second_stage=False, use_endtoend=False, result_path=None, predict_test=False, ckpt_path=None, ref_detfile=None, pickle_result=True, measure_time=False, batch_size=None): model_dir = pathlib.Path(model_dir) if predict_test: result_name = 'predict_test_0095' else: result_name = 'eval_results' if result_path is None: result_path = model_dir / result_name else: result_path = pathlib.Path(result_path) config = pipeline_pb2.TrainEvalPipelineConfig() with open(config_path, "r") as f: proto_str = f.read() text_format.Merge(proto_str, config) input_cfg = config.eval_input_reader model_cfg = config.model.second train_cfg = config.train_config center_limit_range = model_cfg.post_center_limit_range ###################### # BUILD VOXEL GENERATOR ###################### voxel_generator = voxel_builder.build(model_cfg.voxel_generator) bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]] box_coder = box_coder_builder.build(model_cfg.box_coder) target_assigner_cfg = model_cfg.target_assigner target_assigner = target_assigner_builder.build(target_assigner_cfg, bv_range, box_coder) class_names = target_assigner.classes if use_second_stage: net = second_2stage_builder.build(model_cfg, voxel_generator, target_assigner, measure_time=measure_time) elif use_endtoend: net = second_endtoend_builder.build(model_cfg, voxel_generator, target_assigner, measure_time=measure_time) else: net = second_builder.build(model_cfg, voxel_generator, target_assigner, measure_time=measure_time) net.cuda() ######################################### # net = torch.nn.DataParallel(net) ######################################### if ckpt_path is None: torchplus.train.try_restore_latest_checkpoints(model_dir, [net]) else: torchplus.train.restore(ckpt_path, net) if train_cfg.enable_mixed_precision: net.half() print("half inference!") net.metrics_to_float() net.convert_norm_to_float(net) batch_size = batch_size or input_cfg.batch_size eval_dataset = input_reader_builder_tr.build( input_cfg, model_cfg, training=False, voxel_generator=voxel_generator, target_assigner=target_assigner) eval_dataloader = torch.utils.data.DataLoader( eval_dataset, batch_size=batch_size, shuffle=False, num_workers=0,# input_cfg.num_workers, pin_memory=False, collate_fn=merge_second_batch) if train_cfg.enable_mixed_precision: float_dtype = torch.float16 else: float_dtype = torch.float32 net.eval() result_path_step = result_path / f"step_{net.get_global_step()}" result_path_step.mkdir(parents=True, exist_ok=True) t = time.time() dt_annos = [] global_set = None print("Generate output labels...") bar = ProgressBar() bar.start((len(eval_dataset) + batch_size - 1) // batch_size) prep_example_times = [] prep_times = [] t2 = time.time() for example in iter(eval_dataloader): if measure_time: prep_times.append(time.time() - t2) t1 = time.time() torch.cuda.synchronize() example = example_convert_to_torch(example, float_dtype) if measure_time: torch.cuda.synchronize() prep_example_times.append(time.time() - t1) if pickle_result: dt_annos += predict_kitti_to_anno( net, example, class_names, center_limit_range, model_cfg.lidar_input, global_set) else: _predict_kitti_to_file(net, example, result_path_step, class_names, center_limit_range, model_cfg.lidar_input) # print(json.dumps(net.middle_feature_extractor.middle_conv.sparity_dict)) bar.print_bar() if measure_time: t2 = time.time() sec_per_example = len(eval_dataset) / (time.time() - t) print(f'generate label finished({sec_per_example:.2f}/s). start eval:') if measure_time: print(f"avg example to torch time: {np.mean(prep_example_times) * 1000:.3f} ms") print(f"avg prep time: {np.mean(prep_times) * 1000:.3f} ms") for name, val in net.get_avg_time_dict().items(): print(f"avg {name} time = {val * 1000:.3f} ms") if not predict_test: gt_annos = [info["annos"] for info in eval_dataset.dataset.kitti_infos] img_idx = [info["image_idx"] for info in eval_dataset.dataset.kitti_infos] if not pickle_result: dt_annos = kitti.get_label_annos(result_path_step) result = get_official_eval_result(gt_annos, dt_annos, class_names) # print(json.dumps(result, indent=2)) print(result) result = get_coco_eval_result(gt_annos, dt_annos, class_names) print(result) if pickle_result: with open(result_path_step / "result.pkl", 'wb') as f: pickle.dump(dt_annos, f) # annos to txt file if True: os.makedirs(str(result_path_step) + '/txt', exist_ok=True) for i in range(len(dt_annos)): dt_annos[i]['dimensions'] = dt_annos[i]['dimensions'][:, [1, 2, 0]] result_lines = kitti.annos_to_kitti_label(dt_annos[i]) image_idx = img_idx[i] with open(str(result_path_step) + '/txt/%06d.txt' % image_idx, 'w') as f: for result_line in result_lines: f.write(result_line + '\n') abcd = 1 else: os.makedirs(str(result_path_step) + '/txt', exist_ok=True) img_idx = [info["image_idx"] for info in eval_dataset.dataset.kitti_infos] for i in range(len(dt_annos)): dt_annos[i]['dimensions'] = dt_annos[i]['dimensions'][:, [1, 2, 0]] result_lines = kitti.annos_to_kitti_label(dt_annos[i]) image_idx = img_idx[i] with open(str(result_path_step) + '/txt/%06d.txt' % image_idx, 'w') as f: for result_line in result_lines: f.write(result_line + '\n')
def evaluation_from_kitti_dets(self, dt_annos, output_dir): if "annos" not in self._kitti_infos[0]: return None gt_annos = [info["annos"] for info in self._kitti_infos] # firstly convert standard detection to kitti-format dt annos z_axis = 1 # KITTI camera format use y as regular "z" axis. z_center = 1.0 # KITTI camera box's center is [0.5, 1, 0.5] # for regular raw lidar data, z_axis = 2, z_center = 0.5. result_official_dict = get_official_eval_result( gt_annos, dt_annos, self._class_names, z_axis=z_axis, z_center=z_center) result_coco = get_coco_eval_result( gt_annos, dt_annos, self._class_names, z_axis=z_axis, z_center=z_center) # feature extraction for info, det in tqdm(zip(self._kitti_infos, dt_annos), desc="feature", total=len(dt_annos)): pc_info = info["point_cloud"] image_info = info["image"] calib = info["calib"] num_features = pc_info["num_features"] v_path = self._root_path / pc_info["velodyne_path"] v_path = str(v_path.parent.parent / (v_path.parent.stem + "_reduced") / v_path.name) points_v = np.fromfile( v_path, dtype=np.float32, count=-1).reshape([-1, num_features]) rect = calib['R0_rect'] Trv2c = calib['Tr_velo_to_cam'] P2 = calib['P2'] if False: # No longer you need remove outside image-rect (*_reduced pointcloud is already filtered.) points_v = box_np_ops.remove_outside_points( points_v, rect, Trv2c, P2, image_info["image_shape"]) annos = det num_obj = len([n for n in annos['name'] if n != 'DontCare']) # annos = kitti.filter_kitti_anno(annos, ['DontCare']) dims = annos['dimensions'][:num_obj] loc = annos['location'][:num_obj] rots = annos['rotation_y'][:num_obj] gt_boxes_camera = np.concatenate([loc, dims, rots[..., np.newaxis]], axis=1) gt_boxes_lidar = box_np_ops.box_camera_to_lidar( gt_boxes_camera, rect, Trv2c) indices = box_np_ops.points_in_rbbox(points_v[:, :3], gt_boxes_lidar) num_points_in_gt = indices.sum(0) num_ignored = len(annos['dimensions']) - num_obj num_points_in_gt = np.concatenate( [num_points_in_gt, -np.ones([num_ignored])]) annos["num_points_in_det"] = num_points_in_gt.astype(np.int32) return { "results": { "official": result_official_dict["result"], "coco": result_coco["result"], }, "detail": { "eval.kitti": { "official": result_official_dict["detail"], "coco": result_coco["detail"] } }, "result_kitti": result_official_dict["detections"], }
def evaluate(config_path, model_dir, result_path=None, predict_test=False, ckpt_path=None, ref_detfile=None, pickle_result=True, measure_time=False, batch_size=None): model_dir = pathlib.Path(model_dir) print("Predict_test: ", predict_test) if predict_test: result_name = 'predict_test' else: result_name = 'eval_results' if result_path is None: result_path = model_dir / result_name else: result_path = pathlib.Path(result_path) config = pipeline_pb2.TrainEvalPipelineConfig() with open(config_path, "r") as f: proto_str = f.read() text_format.Merge(proto_str, config) input_cfg = config.eval_input_reader model_cfg = config.model.second train_cfg = config.train_config detection_2d_path = config.train_config.detection_2d_path center_limit_range = model_cfg.post_center_limit_range ###################### # BUILD VOXEL GENERATOR ###################### voxel_generator = voxel_builder.build(model_cfg.voxel_generator) bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]] box_coder = box_coder_builder.build(model_cfg.box_coder) target_assigner_cfg = model_cfg.target_assigner target_assigner = target_assigner_builder.build(target_assigner_cfg, bv_range, box_coder) class_names = target_assigner.classes # this one is used for training car detector net = build_inference_net('./configs/car.fhd.config', '../model_dir') fusion_layer = fusion.fusion() fusion_layer.cuda() net.cuda() ############ restore parameters for fusion layer if ckpt_path is None: print("load existing model for fusion layer") torchplus.train.try_restore_latest_checkpoints(model_dir, [fusion_layer]) else: torchplus.train.restore(ckpt_path, fusion_layer) if train_cfg.enable_mixed_precision: net.half() net.metrics_to_float() net.convert_norm_to_float(net) batch_size = batch_size or input_cfg.batch_size eval_dataset = input_reader_builder.build(input_cfg, model_cfg, training=not predict_test, voxel_generator=voxel_generator, target_assigner=target_assigner) eval_dataloader = torch.utils.data.DataLoader( eval_dataset, batch_size=batch_size, shuffle=False, num_workers=0, # input_cfg.num_workers, pin_memory=False, collate_fn=merge_second_batch) if train_cfg.enable_mixed_precision: float_dtype = torch.float16 else: float_dtype = torch.float32 net.eval() fusion_layer.eval() result_path_step = result_path / f"step_{net.get_global_step()}" result_path_step.mkdir(parents=True, exist_ok=True) t = time.time() dt_annos = [] global_set = None print("Generate output labels...") bar = ProgressBar() bar.start((len(eval_dataset) + batch_size - 1) // batch_size) prep_example_times = [] prep_times = [] t2 = time.time() val_loss_final = 0 for example in iter(eval_dataloader): if measure_time: prep_times.append(time.time() - t2) t1 = time.time() torch.cuda.synchronize() example = example_convert_to_torch(example, float_dtype) if measure_time: torch.cuda.synchronize() prep_example_times.append(time.time() - t1) if pickle_result: dt_annos_i, val_losses = predict_kitti_to_anno( net, detection_2d_path, fusion_layer, example, class_names, center_limit_range, model_cfg.lidar_input, global_set) dt_annos += dt_annos_i val_loss_final = val_loss_final + val_losses else: _predict_kitti_to_file(net, detection_2d_path, fusion_layer, example, result_path_step, class_names, center_limit_range, model_cfg.lidar_input) bar.print_bar() if measure_time: t2 = time.time() sec_per_example = len(eval_dataset) / (time.time() - t) print(f'generate label finished({sec_per_example:.2f}/s). start eval:') print("validation_loss:", val_loss_final / len(eval_dataloader)) if measure_time: print( f"avg example to torch time: {np.mean(prep_example_times) * 1000:.3f} ms" ) print(f"avg prep time: {np.mean(prep_times) * 1000:.3f} ms") for name, val in net.get_avg_time_dict().items(): print(f"avg {name} time = {val * 1000:.3f} ms") if not predict_test: gt_annos = [info["annos"] for info in eval_dataset.dataset.kitti_infos] if not pickle_result: dt_annos = kitti.get_label_annos(result_path_step) result = get_official_eval_result(gt_annos, dt_annos, class_names) # print(json.dumps(result, indent=2)) print(result) result = get_coco_eval_result(gt_annos, dt_annos, class_names) print(result) if pickle_result: with open(result_path_step / "result.pkl", 'wb') as f: pickle.dump(dt_annos, f) else: if pickle_result: with open(result_path_step / "result.pkl", 'wb') as f: pickle.dump(dt_annos, f)
def train(config_path, model_dir, result_path=None, create_folder=False, display_step=50, summary_step=5, pickle_result=True, patchs=None): torch.manual_seed(3) np.random.seed(3) if create_folder: if pathlib.Path(model_dir).exists(): model_dir = torchplus.train.create_folder(model_dir) patchs = patchs or [] model_dir = pathlib.Path(model_dir) model_dir.mkdir(parents=True, exist_ok=True) if result_path is None: result_path = model_dir / 'results' config = pipeline_pb2.TrainEvalPipelineConfig() with open(config_path, "r") as f: proto_str = f.read() text_format.Merge(proto_str, config) input_cfg = config.train_input_reader eval_input_cfg = config.eval_input_reader model_cfg = config.model.second train_cfg = config.train_config detection_2d_path = config.train_config.detection_2d_path print("2d detection path:", detection_2d_path) center_limit_range = model_cfg.post_center_limit_range voxel_generator = voxel_builder.build(model_cfg.voxel_generator) bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]] box_coder = box_coder_builder.build(model_cfg.box_coder) target_assigner_cfg = model_cfg.target_assigner target_assigner = target_assigner_builder.build(target_assigner_cfg, bv_range, box_coder) class_names = target_assigner.classes net = build_inference_net('./configs/car.fhd.config', '../model_dir') fusion_layer = fusion.fusion() fusion_layer.cuda() optimizer_cfg = train_cfg.optimizer if train_cfg.enable_mixed_precision: net.half() net.metrics_to_float() net.convert_norm_to_float(net) loss_scale = train_cfg.loss_scale_factor mixed_optimizer = optimizer_builder.build( optimizer_cfg, fusion_layer, mixed=train_cfg.enable_mixed_precision, loss_scale=loss_scale) optimizer = mixed_optimizer # must restore optimizer AFTER using MixedPrecisionWrapper torchplus.train.try_restore_latest_checkpoints(model_dir, [mixed_optimizer]) lr_scheduler = lr_scheduler_builder.build(optimizer_cfg, optimizer, train_cfg.steps) if train_cfg.enable_mixed_precision: float_dtype = torch.float16 else: float_dtype = torch.float32 ###################### # PREPARE INPUT ###################### dataset = input_reader_builder.build(input_cfg, model_cfg, training=True, voxel_generator=voxel_generator, target_assigner=target_assigner) eval_dataset = input_reader_builder.build( eval_input_cfg, model_cfg, training=True, #if rhnning for test, here it needs to be False voxel_generator=voxel_generator, target_assigner=target_assigner) def _worker_init_fn(worker_id): time_seed = np.array(time.time(), dtype=np.int32) np.random.seed(time_seed + worker_id) print(f"WORKER {worker_id} seed:", np.random.get_state()[1][0]) dataloader = torch.utils.data.DataLoader(dataset, batch_size=input_cfg.batch_size, shuffle=True, num_workers=input_cfg.num_workers, pin_memory=False, collate_fn=merge_second_batch, worker_init_fn=_worker_init_fn) eval_dataloader = torch.utils.data.DataLoader( eval_dataset, batch_size=eval_input_cfg.batch_size, shuffle=False, num_workers=eval_input_cfg.num_workers, pin_memory=False, collate_fn=merge_second_batch) data_iter = iter(dataloader) ###################### # TRAINING ###################### focal_loss = SigmoidFocalClassificationLoss() cls_loss_sum = 0 training_detail = [] log_path = model_dir / 'log.txt' training_detail_path = model_dir / 'log.json' if training_detail_path.exists(): with open(training_detail_path, 'r') as f: training_detail = json.load(f) logf = open(log_path, 'a') logf.write(proto_str) logf.write("\n") summary_dir = model_dir / 'summary' summary_dir.mkdir(parents=True, exist_ok=True) writer = SummaryWriter(str(summary_dir)) total_step_elapsed = 0 remain_steps = train_cfg.steps - net.get_global_step() t = time.time() ckpt_start_time = t total_loop = train_cfg.steps // train_cfg.steps_per_eval + 1 #print("steps, steps_per_eval, total_loop:", train_cfg.steps, train_cfg.steps_per_eval, total_loop) # total_loop = remain_steps // train_cfg.steps_per_eval + 1 clear_metrics_every_epoch = train_cfg.clear_metrics_every_epoch net.set_global_step(torch.tensor([0])) if train_cfg.steps % train_cfg.steps_per_eval == 0: total_loop -= 1 mixed_optimizer.zero_grad() try: for _ in range(total_loop): if total_step_elapsed + train_cfg.steps_per_eval > train_cfg.steps: steps = train_cfg.steps % train_cfg.steps_per_eval else: steps = train_cfg.steps_per_eval for step in range(steps): lr_scheduler.step(net.get_global_step()) try: example = next(data_iter) except StopIteration: print("end epoch") if clear_metrics_every_epoch: net.clear_metrics() data_iter = iter(dataloader) example = next(data_iter) example_torch = example_convert_to_torch(example, float_dtype) batch_size = example["anchors"].shape[0] all_3d_output_camera_dict, all_3d_output, top_predictions, fusion_input, tensor_index = net( example_torch, detection_2d_path) d3_gt_boxes = example_torch["d3_gt_boxes"][0, :, :] if d3_gt_boxes.shape[0] == 0: target_for_fusion = np.zeros((1, 70400, 1)) positives = torch.zeros(1, 70400).type(torch.float32).cuda() negatives = torch.zeros(1, 70400).type(torch.float32).cuda() negatives[:, :] = 1 else: d3_gt_boxes_camera = box_torch_ops.box_lidar_to_camera( d3_gt_boxes, example_torch['rect'][0, :], example_torch['Trv2c'][0, :]) d3_gt_boxes_camera_bev = d3_gt_boxes_camera[:, [ 0, 2, 3, 5, 6 ]] ###### predicted bev boxes pred_3d_box = all_3d_output_camera_dict[0]["box3d_camera"] pred_bev_box = pred_3d_box[:, [0, 2, 3, 5, 6]] #iou_bev = bev_box_overlap(d3_gt_boxes_camera_bev.detach().cpu().numpy(), pred_bev_box.detach().cpu().numpy(), criterion=-1) iou_bev = d3_box_overlap( d3_gt_boxes_camera.detach().cpu().numpy(), pred_3d_box.squeeze().detach().cpu().numpy(), criterion=-1) iou_bev_max = np.amax(iou_bev, axis=0) #print(np.max(iou_bev_max)) target_for_fusion = ((iou_bev_max >= 0.7) * 1).reshape( 1, -1, 1) positive_index = ((iou_bev_max >= 0.7) * 1).reshape(1, -1) positives = torch.from_numpy(positive_index).type( torch.float32).cuda() negative_index = ((iou_bev_max <= 0.5) * 1).reshape(1, -1) negatives = torch.from_numpy(negative_index).type( torch.float32).cuda() cls_preds, flag = fusion_layer(fusion_input.cuda(), tensor_index.cuda()) one_hot_targets = torch.from_numpy(target_for_fusion).type( torch.float32).cuda() negative_cls_weights = negatives.type(torch.float32) * 1.0 cls_weights = negative_cls_weights + 1.0 * positives.type( torch.float32) pos_normalizer = positives.sum(1, keepdim=True).type( torch.float32) cls_weights /= torch.clamp(pos_normalizer, min=1.0) if flag == 1: cls_losses = focal_loss._compute_loss( cls_preds, one_hot_targets, cls_weights.cuda()) # [N, M] cls_losses_reduced = cls_losses.sum( ) / example_torch['labels'].shape[0] cls_loss_sum = cls_loss_sum + cls_losses_reduced if train_cfg.enable_mixed_precision: loss *= loss_scale cls_losses_reduced.backward() mixed_optimizer.step() mixed_optimizer.zero_grad() net.update_global_step() step_time = (time.time() - t) t = time.time() metrics = {} global_step = net.get_global_step() if global_step % display_step == 0: print("now it is", global_step, "steps", " and the cls_loss is :", cls_loss_sum / display_step, "learning_rate: ", float(optimizer.lr), file=logf) print("now it is", global_step, "steps", " and the cls_loss is :", cls_loss_sum / display_step, "learning_rate: ", float(optimizer.lr)) cls_loss_sum = 0 ckpt_elasped_time = time.time() - ckpt_start_time if ckpt_elasped_time > train_cfg.save_checkpoints_secs: torchplus.train.save_models(model_dir, [fusion_layer, optimizer], net.get_global_step()) ckpt_start_time = time.time() total_step_elapsed += steps torchplus.train.save_models(model_dir, [fusion_layer, optimizer], net.get_global_step()) fusion_layer.eval() net.eval() result_path_step = result_path / f"step_{net.get_global_step()}" result_path_step.mkdir(parents=True, exist_ok=True) print("#################################") print("#################################", file=logf) print("# EVAL") print("# EVAL", file=logf) print("#################################") print("#################################", file=logf) print("Generate output labels...") print("Generate output labels...", file=logf) t = time.time() dt_annos = [] prog_bar = ProgressBar() net.clear_timer() prog_bar.start( (len(eval_dataset) + eval_input_cfg.batch_size - 1) // eval_input_cfg.batch_size) val_loss_final = 0 for example in iter(eval_dataloader): example = example_convert_to_torch(example, float_dtype) if pickle_result: dt_annos_i, val_losses = predict_kitti_to_anno( net, detection_2d_path, fusion_layer, example, class_names, center_limit_range, model_cfg.lidar_input) dt_annos += dt_annos_i val_loss_final = val_loss_final + val_losses else: _predict_kitti_to_file(net, detection_2d_path, example, result_path_step, class_names, center_limit_range, model_cfg.lidar_input) prog_bar.print_bar() sec_per_ex = len(eval_dataset) / (time.time() - t) print("validation_loss:", val_loss_final / len(eval_dataloader)) print("validation_loss:", val_loss_final / len(eval_dataloader), file=logf) print(f'generate label finished({sec_per_ex:.2f}/s). start eval:') print(f'generate label finished({sec_per_ex:.2f}/s). start eval:', file=logf) gt_annos = [ info["annos"] for info in eval_dataset.dataset.kitti_infos ] if not pickle_result: dt_annos = kitti.get_label_annos(result_path_step) # result = get_official_eval_result_v2(gt_annos, dt_annos, class_names) result = get_official_eval_result(gt_annos, dt_annos, class_names) print(result, file=logf) print(result) writer.add_text('eval_result', json.dumps(result, indent=2), global_step) result = get_coco_eval_result(gt_annos, dt_annos, class_names) print(result, file=logf) print(result) if pickle_result: with open(result_path_step / "result.pkl", 'wb') as f: pickle.dump(dt_annos, f) writer.add_text('eval_result', result, global_step) #net.train() fusion_layer.train() except Exception as e: torchplus.train.save_models(model_dir, [fusion_layer, optimizer], net.get_global_step()) logf.close() raise e # save model before exit torchplus.train.save_models(model_dir, [fusion_layer, optimizer], net.get_global_step()) logf.close()