def build_network_(): global BACKEND instance = request.json cfg_path = Path(instance["config_path"]) ckpt_path = Path(instance["checkpoint_path"]) response = {"status": "normal"} if BACKEND.root_path is None: return error_response("root path is not set") if not cfg_path.exists(): return error_response("config file not exist.") if not ckpt_path.exists(): return error_response("ckpt file not exist.") config = pipeline_pb2.TrainEvalPipelineConfig() with open(cfg_path, "r") as f: proto_str = f.read() text_format.Merge(proto_str, config) device = device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") net = build_network(config.model.second).to(device).float().eval() net.load_state_dict(torch.load(ckpt_path)) eval_input_cfg = config.eval_input_reader BACKEND.dataset = input_reader_builder.build( eval_input_cfg, config.model.second, training=False, voxel_generator=net.voxel_generator, target_assigner=net.target_assigner).dataset BACKEND.net = net BACKEND.config = config BACKEND.device = device response = jsonify(results=[response]) response.headers['Access-Control-Allow-Headers'] = '*' print("build_network successful!") return response
def trans_onnx(config_path, ckpt_path): device = torch.device("cuda" if torch.cuda.is_available() else "cpu") config = pipeline_pb2.TrainEvalPipelineConfig() with open(config_path, "r") as f: proto_str = f.read() text_format.Merge(proto_str, config) model_cfg = config.model.second net = build_network(model_cfg, measure_time=False).to(device) net.load_state_dict(torch.load(ckpt_path)) voxels = torch.ones([12000, 100, 4], dtype=torch.float32, device=device) num_points = torch.ones([12000], dtype=torch.float32, device=device) coors = torch.ones([12000, 4], dtype=torch.float32, device=device) example1 = (voxels, num_points, coors) spatial_features = torch.ones([1, 64, 496, 432], dtype=torch.float32, device=device) example2 = (spatial_features, ) torch.onnx.export(net.voxel_feature_extractor, example1, "pfe.onnx", verbose=False) torch.onnx.export(net.rpn, example2, "rpn.onnx", verbose=False)
def _init_model(self): self.config = pipeline_pb2.TrainEvalPipelineConfig() with open(self.config_p, 'r') as f: proto_str = f.read() text_format.Merge(proto_str, self.config) self.input_cfg = self.config.eval_input_reader self.model_cfg = self.config.model.second config_tool.change_detection_range_v2(self.model_cfg, [-50, -50, 50, 50]) logging.info('config loaded.') self.net = build_network(self.model_cfg).to(device).eval() self.net.load_state_dict(torch.load(self.model_p)) self.target_assigner = self.net.target_assigner self.voxel_generator = self.net.voxel_generator logging.info('network done, voxel done.') grid_size = self.voxel_generator.grid_size feature_map_size = grid_size[:2] // config_tool.get_downsample_factor( self.model_cfg) feature_map_size = [*feature_map_size, 1][::-1] self.anchors = self.target_assigner.generate_anchors( feature_map_size)['anchors'] self.anchors = torch.tensor(self.anchors, dtype=torch.float32, device=device) self.anchors = self.anchors.view(1, -1, 7) logging.info('anchors generated.')
def pytorch_inference(config_path=None, ckpt_path=None, data_path=None): config = pipeline_pb2.TrainEvalPipelineConfig() with open(config_path, "r") as f: proto_str = f.read() text_format.Merge(proto_str, config) model_cfg = config.model.second device = torch.device("cuda" if torch.cuda.is_available() else "cpu") net = build_network(model_cfg).to(device).eval() net.load_state_dict(torch.load(ckpt_path)) points = read_pointcloud(data_path) example = generate_example(net, model_cfg, points, device) pred = net(example)[0] boxes_lidar = pred["box3d_lidar"].detach().cpu().numpy() vis_voxel_size = [0.1, 0.1, 0.1] vis_point_range = [-50, -30, -3, 50, 30, 1] bev_map = simplevis.point_to_vis_bev(points, vis_voxel_size, vis_point_range) bev_map = simplevis.draw_box_in_bev(bev_map, vis_point_range, boxes_lidar, [0, 255, 0], 2) plt.imsave("result.png", bev_map)
def onnx_inference(config_path, data_path, pfe_path, rpn_path): config = pipeline_pb2.TrainEvalPipelineConfig() with open(config_path, "r") as f: proto_str = f.read() text_format.Merge(proto_str, config) model_cfg = config.model.second device = torch.device("cuda" if torch.cuda.is_available() else "cpu") net = build_network(model_cfg).to(device).eval() points = read_pointcloud(data_path) example = generate_example(net, model_cfg, points, device) #onnx inference ort_session_pfe = onnxruntime.InferenceSession(pfe_path) ort_session_rpn = onnxruntime.InferenceSession(rpn_path) # compute ONNX Runtime output prediction ort_inputs_pfe = { ort_session_pfe.get_inputs()[0].name: to_numpy(example["voxels"]), ort_session_pfe.get_inputs()[1].name: to_numpy(example["num_points"]), ort_session_pfe.get_inputs()[2].name: to_numpy(example["coordinates"]) } ort_outs_pfe = ort_session_pfe.run(None, ort_inputs_pfe) voxel_features = torch.from_numpy(ort_outs_pfe[0]).to(device) spatial_features = net.middle_feature_extractor(voxel_features, example["coordinates"], 1) ort_inputs_rpn = { ort_session_rpn.get_inputs()[0].name: to_numpy(spatial_features) } ort_outs_rpn = ort_session_rpn.run(None, ort_inputs_rpn) preds_dict = {} preds_dict["box_preds"] = torch.from_numpy(ort_outs_rpn[0]).to(device) preds_dict["cls_preds"] = torch.from_numpy(ort_outs_rpn[1]).to(device) preds_dict["dir_cls_preds"] = torch.from_numpy(ort_outs_rpn[2]).to(device) with torch.no_grad(): pred = net.predict(example, preds_dict)[0] boxes_lidar = pred["box3d_lidar"].detach().cpu().numpy() vis_voxel_size = [0.1, 0.1, 0.1] vis_point_range = [-50, -30, -3, 50, 30, 1] bev_map = simplevis.point_to_vis_bev(points, vis_voxel_size, vis_point_range) bev_map = simplevis.draw_box_in_bev(bev_map, vis_point_range, boxes_lidar, [0, 255, 0], 2) plt.imsave("result_onnx.png", bev_map)
def main(): cfg_path = Path('/..../pointpillars/car/xyres_##.config') ckpt_path = Path('/..../voxelnet-######.tckpt') config = pipeline_pb2.TrainEvalPipelineConfig() print("config reading") with open(cfg_path, "r") as f: proto_str = f.read() text_format.Merge(proto_str, config) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") print("building net") net = build_network(config.model.second).to(device).float().eval() net.load_state_dict(torch.load(ckpt_path)) print("net built") eval_input_cfg = config.eval_input_reader dataset = input_reader_builder.build( eval_input_cfg, config.model.second, training=False, voxel_generator=net.voxel_generator, target_assigner=net.target_assigner).dataset idx = 0 example = dataset[idx] example["coordinates"] = np.pad(example["coordinates"], ((0, 0), (1, 0)), mode='constant', constant_values=0) # don't forget to add newaxis for anchors example["anchors"] = example["anchors"][np.newaxis, ...] example_torch = example_convert_to_torch(example, device=device) voxels = example_torch["voxels"] num_points = example_torch["num_points"] coors = example_torch["coordinates"] batch_anchors = example["anchors"] batch_size_dev = batch_anchors.shape[0] voxel_features = net.voxel_feature_extractor(voxels, num_points, coors) spatial_features = net.middle_feature_extractor(voxel_features, coors, batch_size_dev) # Export the model print("exporting as onnx") torch_out = torch.onnx._export(net.rpn, (spatial_features), "rpn.onnx", export_params=True) print("export complete")
def __init__(self, config_filepath, weight_filepath): # ====================================================== # Read Config file # ====================================================== self.config = pipeline_pb2.TrainEvalPipelineConfig() with open(config_filepath, "r") as f: proto_str = f.read() text_format.Merge(proto_str, self.config) self.input_cfg = self.config.eval_input_reader self.model_cfg = self.config.model.second # config_tool.change_detection_range_v2(self.model_cfg, [-50, -50, 50, 50]) # ====================================================== # Build Network, Target Assigner and Voxel Generator # ====================================================== self.device = torch.device( "cuda" if torch.cuda.is_available() else "cpu") self.net = build_network(self.model_cfg).to(self.device).eval() self.net.load_state_dict(torch.load(weight_filepath)) self.target_assigner = self.net.target_assigner self.voxel_generator = self.net.voxel_generator # ====================================================== # Generate Anchors # ====================================================== grid_size = self.voxel_generator.grid_size print("========= grid_size") print(grid_size) print("========= voxel_size") print(self.voxel_generator.voxel_size) print("========= point_cloud_range") print(self.voxel_generator.point_cloud_range) feature_map_size = grid_size[:2] // config_tool.get_downsample_factor( self.model_cfg) feature_map_size = [*feature_map_size, 1][::-1] print("========= feature_map_size") print(feature_map_size) self.anchors = self.target_assigner.generate_anchors( feature_map_size)["anchors"] self.anchors = torch.tensor(self.anchors, dtype=torch.float32, device=self.device) self.anchors = self.anchors.view(1, -1, 7) print("========= anchors.shape") print(self.anchors.shape)
def build_network(self): self.net = build_network(self.model_cfg).to(self.device).eval() self.net.load_state_dict(torch.load(self.model_path)) self.target_assigner = self.net.target_assigner self.voxel_generator = self.net.voxel_generator
idx = np.where(scores > threshold)[0] # filter low score ones box3d = box3d[idx, :] # label is one-dim labels = np.take(labels, idx) scores = np.take(scores, idx) pred['box3d_lidar'] = box3d pred['scores'] = scores pred['label_preds'] = labels return pred # In[9]: ckpt_path = "/home/ags/second_test/all_fhd.30/voxelnet-29369.tckpt" net = build_network(config.model.second).to(device).float().eval() net.load_state_dict(torch.load(ckpt_path)) eval_input_cfg = config.eval_input_reader eval_input_cfg.dataset.kitti_root_path = root_path eval_input_cfg.dataset.kitti_info_path = info_path dataset = input_reader_builder.build( eval_input_cfg, config.model.second, training=False, voxel_generator=net.voxel_generator, target_assigner=net.target_assigner) #.dataset batch_size = 4 num_workers = 4 dataloader = torch.utils.data.DataLoader(
with open(config_path, "r") as f: proto_str = f.read() text_format.Merge(proto_str, config) # input_cfg = config.eval_input_reader model_cfg = config.model.second # config_tool.change_detection_range(model_cfg, [-50, -50, 50, 50]) # 显存不够 config_tool.change_detection_range(model_cfg, [-70, -32, 70, 32]) # 可以运行 # config_tool.change_detection_range(model_cfg, [x, y, x, y]) # 可以运行 device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # device = torch.device("cpu") t1 = time.time() # Build Network, Target Assigner and Voxel Generator # ckpt_path = "/home/ogailab/autoware.ai.10.0/src/autoware/core_perception/lidar_second/70m_cb_all/voxelnet-148480.tckpt" ckpt_path = "/home/ogailab/tiatia/codes/TALite/model3.0/nuscene/all/fhd.rpnv2/voxelnet-140670.tckpt" net = build_network(model_cfg).to(device).eval() net.load_state_dict(torch.load(ckpt_path)) target_assigner = net.target_assigner voxel_generator = net.voxel_generator t2 = time.time() # Generate Anchors grid_size = voxel_generator.grid_size feature_map_size = grid_size[:2] // config_tool.get_downsample_factor( model_cfg) feature_map_size = [*feature_map_size, 1][::-1] anchors = target_assigner.generate_anchors(feature_map_size)["anchors"] anchors = torch.tensor(anchors, dtype=torch.float32, device=device) anchors = anchors.view(1, -1, 7) t3 = time.time()
def main(config_path, lc_horizon, num_examples, model_dir, ckpt_path=None, **kwargs): """Don't support pickle_result anymore. if you want to generate kitti label file, please use kitti_anno_to_label_file and convert_detection_to_kitti_annos in second.data.kitti_dataset. """ assert len(kwargs) == 0 model_dir = str(Path(model_dir).resolve()) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") if isinstance(config_path, str): # directly provide a config object. this usually used # when you want to eval with several different parameters in # one script. config = pipeline_pb2.TrainEvalPipelineConfig() with open(config_path, "r") as f: proto_str = f.read() text_format.Merge(proto_str, config) else: config = config_path input_cfg = config.eval_input_reader input_cfg.cum_lc_wrapper.lc_horizon = lc_horizon model_cfg = config.model.second train_cfg = config.train_config net = build_network(model_cfg, measure_time=False).to(device) if train_cfg.enable_mixed_precision: net.half() print("half inference!") net.metrics_to_float() net.convert_norm_to_float(net) target_assigner = net.target_assigner voxel_generator = net.voxel_generator if ckpt_path is None: assert model_dir is not None torchplus.train.try_restore_latest_checkpoints(model_dir, [net]) else: torchplus.train.restore(ckpt_path, net) batch_size = 1 eval_dataset = input_reader_builder.build(input_cfg, model_cfg, training=False, voxel_generator=voxel_generator, target_assigner=target_assigner, net=net) if train_cfg.enable_mixed_precision: float_dtype = torch.float16 else: float_dtype = torch.float32 net.eval() t = time.time() detections = [] print("Generate output labels...") bar = ProgressBar() bar.start((len(eval_dataset) + batch_size - 1) // batch_size) prep_example_times = [] prep_times = [] t2 = time.time() times = [] for scene_id in trange(num_examples): idx = eval_dataset.scene_id_and_step_to_idx(scene_id, lc_horizon) torch.cuda.synchronize() b_ex_time = time.time() example = eval_dataset[idx] example = merge_second_batch([example]) example = example_convert_to_torch(example, float_dtype) with torch.no_grad(): detections = net(example) torch.cuda.synchronize() e_ex_time = time.time() del example, detections times.append(e_ex_time - b_ex_time) times = np.array(times) mean = times.mean() interval = 1.96 * times.std() / np.sqrt( len(times)) # 95% confidence interval return mean, interval
def build_net_v2(ckpt_path, model_cfg, device): # returns the targer assigner, voxel generator net = build_network(model_cfg).to(device).eval() net.load_state_dict(torch.load(ckpt_path)) return net.target_assigner, net.voxel_generator, device, net
def detect(scene_token, config_path, ckpt_path, info_path, root_path, result_path): ### Read Config file torch.set_num_threads(2) #config_path = "configs/nuscenes/all.pp.lowa_large_range_v2.config" config = pipeline_pb2.TrainEvalPipelineConfig() with open(config_path, "r") as f: proto_str = f.read() text_format.Merge(proto_str, config) input_cfg = config.eval_input_reader model_cfg = config.model.second # config_tool.change_detection_range_v2(model_cfg, [-50, -50, 50, 50]) device = torch.device("cuda" if torch.cuda.is_available() else "cpu") ### Build Network, Target Assigner and Voxel Generator #info_path = '/home/itiv/Desktop/lyft-dataset/infos_val.pkl' #root_path = '/home/itiv/Desktop/lyft-dataset' with open(info_path, 'rb') as f: infos = pickle.load(f) token2info = {} for info in infos['infos']: token2info[info['token']] = info #ckpt_path = "/home/itiv/Desktop/repo/scenarios_in_CarMaker/BA_Daniel/Lyft-Detector/second.pytorch/second/model/model_large_range_v2/voxelnet-33445.tckpt" net = build_network(config.model.second).to(device).float().eval() net.load_state_dict(torch.load(ckpt_path)) eval_input_cfg = config.eval_input_reader eval_input_cfg.dataset.kitti_root_path = root_path eval_input_cfg.dataset.kitti_info_path = info_path dataset = input_reader_builder.build( eval_input_cfg, config.model.second, training=False, voxel_generator=net.voxel_generator, target_assigner=net.target_assigner) #.dataset batch_size = 2 num_workers = 2 dataloader = torch.utils.data.DataLoader(dataset, batch_size=batch_size, shuffle=False, num_workers=num_workers, pin_memory=False, collate_fn=merge_second_batch) target_assigner = net.target_assigner voxel_generator = net.voxel_generator classes = target_assigner.classes detections = [] #tk0 = prog_bar(dataloader, total=len(dataloader)) tk0 = (dataloader) for idx, examples in enumerate(tk0): #print(idx) #print(examples) try: example_torch = example_convert_to_torch(examples, device=device) detections += net(example_torch) except Exception as e: print(e) import pdb pdb.set_trace() threshold = 0.2 first_sample_token = detections[0]['metadata']['token'] dict_detections = {"results": {}} for idx, pred in enumerate((detections)): pred = thresholded_pred(pred, threshold) #token = tokens[idx]['token'] token = pred['metadata']['token'] dict_detections['results'].update( get_pred_dict(pred, token, classes, token2info)) #pred_str = get_pred_str(pred, token) #predStrings.append(pred_str) #index = df[df['Id'] == token].index[0] #df.loc[index, 'PredictionString'] = pred_str #df.to_csv(f'final.csv', index=False) #print(dict_detections) #path_to_result = f'/home/itiv/Desktop/lyft-dataset/detections-largev2.json' with open(result_path + '/detections_' + scene_token + '.json', 'w') as fp: json.dump(dict_detections, fp)