def build_network_():
global BACKEND
instance = request.json
cfg_path = Path(instance["config_path"])
ckpt_path = Path(instance["checkpoint_path"])
response = {"status": "normal"}
if BACKEND.root_path is None:
return error_response("root path is not set")
if not cfg_path.exists():
return error_response("config file not exist.")
if not ckpt_path.exists():
return error_response("ckpt file not exist.")
config = pipeline_pb2.TrainEvalPipelineConfig()
with open(cfg_path, "r") as f:
proto_str = f.read()
text_format.Merge(proto_str, config)
device = device = torch.device(
"cuda" if torch.cuda.is_available() else "cpu")
net = build_network(config.model.second).to(device).float().eval()
net.load_state_dict(torch.load(ckpt_path))
eval_input_cfg = config.eval_input_reader
BACKEND.dataset = input_reader_builder.build(
eval_input_cfg,
config.model.second,
training=False,
voxel_generator=net.voxel_generator,
target_assigner=net.target_assigner).dataset
BACKEND.net = net
BACKEND.config = config
BACKEND.device = device
response = jsonify(results=[response])
response.headers['Access-Control-Allow-Headers'] = '*'
print("build_network successful!")
return response
def trans_onnx(config_path, ckpt_path):
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
config = pipeline_pb2.TrainEvalPipelineConfig()
with open(config_path, "r") as f:
proto_str = f.read()
text_format.Merge(proto_str, config)
model_cfg = config.model.second
net = build_network(model_cfg, measure_time=False).to(device)
net.load_state_dict(torch.load(ckpt_path))
voxels = torch.ones([12000, 100, 4], dtype=torch.float32, device=device)
num_points = torch.ones([12000], dtype=torch.float32, device=device)
coors = torch.ones([12000, 4], dtype=torch.float32, device=device)
example1 = (voxels, num_points, coors)
spatial_features = torch.ones([1, 64, 496, 432],
dtype=torch.float32,
device=device)
example2 = (spatial_features, )
torch.onnx.export(net.voxel_feature_extractor,
example1,
"pfe.onnx",
verbose=False)
torch.onnx.export(net.rpn, example2, "rpn.onnx", verbose=False)
def _init_model(self):
self.config = pipeline_pb2.TrainEvalPipelineConfig()
with open(self.config_p, 'r') as f:
proto_str = f.read()
text_format.Merge(proto_str, self.config)
self.input_cfg = self.config.eval_input_reader
self.model_cfg = self.config.model.second
config_tool.change_detection_range_v2(self.model_cfg,
[-50, -50, 50, 50])
logging.info('config loaded.')
self.net = build_network(self.model_cfg).to(device).eval()
self.net.load_state_dict(torch.load(self.model_p))
self.target_assigner = self.net.target_assigner
self.voxel_generator = self.net.voxel_generator
logging.info('network done, voxel done.')
grid_size = self.voxel_generator.grid_size
feature_map_size = grid_size[:2] // config_tool.get_downsample_factor(
self.model_cfg)
feature_map_size = [*feature_map_size, 1][::-1]
self.anchors = self.target_assigner.generate_anchors(
feature_map_size)['anchors']
self.anchors = torch.tensor(self.anchors,
dtype=torch.float32,
device=device)
self.anchors = self.anchors.view(1, -1, 7)
logging.info('anchors generated.')
def pytorch_inference(config_path=None, ckpt_path=None, data_path=None):
config = pipeline_pb2.TrainEvalPipelineConfig()
with open(config_path, "r") as f:
proto_str = f.read()
text_format.Merge(proto_str, config)
model_cfg = config.model.second
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
net = build_network(model_cfg).to(device).eval()
net.load_state_dict(torch.load(ckpt_path))
points = read_pointcloud(data_path)
example = generate_example(net, model_cfg, points, device)
pred = net(example)[0]
boxes_lidar = pred["box3d_lidar"].detach().cpu().numpy()
vis_voxel_size = [0.1, 0.1, 0.1]
vis_point_range = [-50, -30, -3, 50, 30, 1]
bev_map = simplevis.point_to_vis_bev(points, vis_voxel_size,
vis_point_range)
bev_map = simplevis.draw_box_in_bev(bev_map, vis_point_range, boxes_lidar,
[0, 255, 0], 2)
plt.imsave("result.png", bev_map)
def onnx_inference(config_path, data_path, pfe_path, rpn_path):
config = pipeline_pb2.TrainEvalPipelineConfig()
with open(config_path, "r") as f:
proto_str = f.read()
text_format.Merge(proto_str, config)
model_cfg = config.model.second
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
net = build_network(model_cfg).to(device).eval()
points = read_pointcloud(data_path)
example = generate_example(net, model_cfg, points, device)
#onnx inference
ort_session_pfe = onnxruntime.InferenceSession(pfe_path)
ort_session_rpn = onnxruntime.InferenceSession(rpn_path)
# compute ONNX Runtime output prediction
ort_inputs_pfe = {
ort_session_pfe.get_inputs()[0].name: to_numpy(example["voxels"]),
ort_session_pfe.get_inputs()[1].name: to_numpy(example["num_points"]),
ort_session_pfe.get_inputs()[2].name: to_numpy(example["coordinates"])
}
ort_outs_pfe = ort_session_pfe.run(None, ort_inputs_pfe)
voxel_features = torch.from_numpy(ort_outs_pfe[0]).to(device)
spatial_features = net.middle_feature_extractor(voxel_features,
example["coordinates"], 1)
ort_inputs_rpn = {
ort_session_rpn.get_inputs()[0].name: to_numpy(spatial_features)
}
ort_outs_rpn = ort_session_rpn.run(None, ort_inputs_rpn)
preds_dict = {}
preds_dict["box_preds"] = torch.from_numpy(ort_outs_rpn[0]).to(device)
preds_dict["cls_preds"] = torch.from_numpy(ort_outs_rpn[1]).to(device)
preds_dict["dir_cls_preds"] = torch.from_numpy(ort_outs_rpn[2]).to(device)
with torch.no_grad():
pred = net.predict(example, preds_dict)[0]
boxes_lidar = pred["box3d_lidar"].detach().cpu().numpy()
vis_voxel_size = [0.1, 0.1, 0.1]
vis_point_range = [-50, -30, -3, 50, 30, 1]
bev_map = simplevis.point_to_vis_bev(points, vis_voxel_size,
vis_point_range)
bev_map = simplevis.draw_box_in_bev(bev_map, vis_point_range, boxes_lidar,
[0, 255, 0], 2)
plt.imsave("result_onnx.png", bev_map)
def main():
cfg_path = Path('/..../pointpillars/car/xyres_##.config')
ckpt_path = Path('/..../voxelnet-######.tckpt')
config = pipeline_pb2.TrainEvalPipelineConfig()
print("config reading")
with open(cfg_path, "r") as f:
proto_str = f.read()
text_format.Merge(proto_str, config)
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print("building net")
net = build_network(config.model.second).to(device).float().eval()
net.load_state_dict(torch.load(ckpt_path))
print("net built")
eval_input_cfg = config.eval_input_reader
dataset = input_reader_builder.build(
eval_input_cfg,
config.model.second,
training=False,
voxel_generator=net.voxel_generator,
target_assigner=net.target_assigner).dataset
idx = 0
example = dataset[idx]
example["coordinates"] = np.pad(example["coordinates"], ((0, 0), (1, 0)),
mode='constant',
constant_values=0)
# don't forget to add newaxis for anchors
example["anchors"] = example["anchors"][np.newaxis, ...]
example_torch = example_convert_to_torch(example, device=device)
voxels = example_torch["voxels"]
num_points = example_torch["num_points"]
coors = example_torch["coordinates"]
batch_anchors = example["anchors"]
batch_size_dev = batch_anchors.shape[0]
voxel_features = net.voxel_feature_extractor(voxels, num_points, coors)
spatial_features = net.middle_feature_extractor(voxel_features, coors,
batch_size_dev)
# Export the model
print("exporting as onnx")
torch_out = torch.onnx._export(net.rpn, (spatial_features),
"rpn.onnx",
export_params=True)
print("export complete")
def __init__(self, config_filepath, weight_filepath):
# ======================================================
# Read Config file
# ======================================================
self.config = pipeline_pb2.TrainEvalPipelineConfig()
with open(config_filepath, "r") as f:
proto_str = f.read()
text_format.Merge(proto_str, self.config)
self.input_cfg = self.config.eval_input_reader
self.model_cfg = self.config.model.second
# config_tool.change_detection_range_v2(self.model_cfg, [-50, -50, 50, 50])
# ======================================================
# Build Network, Target Assigner and Voxel Generator
# ======================================================
self.device = torch.device(
"cuda" if torch.cuda.is_available() else "cpu")
self.net = build_network(self.model_cfg).to(self.device).eval()
self.net.load_state_dict(torch.load(weight_filepath))
self.target_assigner = self.net.target_assigner
self.voxel_generator = self.net.voxel_generator
# ======================================================
# Generate Anchors
# ======================================================
grid_size = self.voxel_generator.grid_size
print("========= grid_size")
print(grid_size)
print("========= voxel_size")
print(self.voxel_generator.voxel_size)
print("========= point_cloud_range")
print(self.voxel_generator.point_cloud_range)
feature_map_size = grid_size[:2] // config_tool.get_downsample_factor(
self.model_cfg)
feature_map_size = [*feature_map_size, 1][::-1]
print("========= feature_map_size")
print(feature_map_size)
self.anchors = self.target_assigner.generate_anchors(
feature_map_size)["anchors"]
self.anchors = torch.tensor(self.anchors,
dtype=torch.float32,
device=self.device)
self.anchors = self.anchors.view(1, -1, 7)
print("========= anchors.shape")
print(self.anchors.shape)
def build_network(self):
self.net = build_network(self.model_cfg).to(self.device).eval()
self.net.load_state_dict(torch.load(self.model_path))
self.target_assigner = self.net.target_assigner
self.voxel_generator = self.net.voxel_generator
idx = np.where(scores > threshold)[0]
# filter low score ones
box3d = box3d[idx, :]
# label is one-dim
labels = np.take(labels, idx)
scores = np.take(scores, idx)
pred['box3d_lidar'] = box3d
pred['scores'] = scores
pred['label_preds'] = labels
return pred
# In[9]:
ckpt_path = "/home/ags/second_test/all_fhd.30/voxelnet-29369.tckpt"
net = build_network(config.model.second).to(device).float().eval()
net.load_state_dict(torch.load(ckpt_path))
eval_input_cfg = config.eval_input_reader
eval_input_cfg.dataset.kitti_root_path = root_path
eval_input_cfg.dataset.kitti_info_path = info_path
dataset = input_reader_builder.build(
eval_input_cfg,
config.model.second,
training=False,
voxel_generator=net.voxel_generator,
target_assigner=net.target_assigner) #.dataset
batch_size = 4
num_workers = 4
dataloader = torch.utils.data.DataLoader(
with open(config_path, "r") as f:
proto_str = f.read()
text_format.Merge(proto_str, config)
# input_cfg = config.eval_input_reader
model_cfg = config.model.second
# config_tool.change_detection_range(model_cfg, [-50, -50, 50, 50]) # 显存不够
config_tool.change_detection_range(model_cfg, [-70, -32, 70, 32]) # 可以运行
# config_tool.change_detection_range(model_cfg, [x, y, x, y]) # 可以运行
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# device = torch.device("cpu")
t1 = time.time()
# Build Network, Target Assigner and Voxel Generator
# ckpt_path = "/home/ogailab/autoware.ai.10.0/src/autoware/core_perception/lidar_second/70m_cb_all/voxelnet-148480.tckpt"
ckpt_path = "/home/ogailab/tiatia/codes/TALite/model3.0/nuscene/all/fhd.rpnv2/voxelnet-140670.tckpt"
net = build_network(model_cfg).to(device).eval()
net.load_state_dict(torch.load(ckpt_path))
target_assigner = net.target_assigner
voxel_generator = net.voxel_generator
t2 = time.time()
# Generate Anchors
grid_size = voxel_generator.grid_size
feature_map_size = grid_size[:2] // config_tool.get_downsample_factor(
model_cfg)
feature_map_size = [*feature_map_size, 1][::-1]
anchors = target_assigner.generate_anchors(feature_map_size)["anchors"]
anchors = torch.tensor(anchors, dtype=torch.float32, device=device)
anchors = anchors.view(1, -1, 7)
t3 = time.time()
def main(config_path,
lc_horizon,
num_examples,
model_dir,
ckpt_path=None,
**kwargs):
"""Don't support pickle_result anymore. if you want to generate kitti label file,
please use kitti_anno_to_label_file and convert_detection_to_kitti_annos
in second.data.kitti_dataset.
"""
assert len(kwargs) == 0
model_dir = str(Path(model_dir).resolve())
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
if isinstance(config_path, str):
# directly provide a config object. this usually used
# when you want to eval with several different parameters in
# one script.
config = pipeline_pb2.TrainEvalPipelineConfig()
with open(config_path, "r") as f:
proto_str = f.read()
text_format.Merge(proto_str, config)
else:
config = config_path
input_cfg = config.eval_input_reader
input_cfg.cum_lc_wrapper.lc_horizon = lc_horizon
model_cfg = config.model.second
train_cfg = config.train_config
net = build_network(model_cfg, measure_time=False).to(device)
if train_cfg.enable_mixed_precision:
net.half()
print("half inference!")
net.metrics_to_float()
net.convert_norm_to_float(net)
target_assigner = net.target_assigner
voxel_generator = net.voxel_generator
if ckpt_path is None:
assert model_dir is not None
torchplus.train.try_restore_latest_checkpoints(model_dir, [net])
else:
torchplus.train.restore(ckpt_path, net)
batch_size = 1
eval_dataset = input_reader_builder.build(input_cfg,
model_cfg,
training=False,
voxel_generator=voxel_generator,
target_assigner=target_assigner,
net=net)
if train_cfg.enable_mixed_precision:
float_dtype = torch.float16
else:
float_dtype = torch.float32
net.eval()
t = time.time()
detections = []
print("Generate output labels...")
bar = ProgressBar()
bar.start((len(eval_dataset) + batch_size - 1) // batch_size)
prep_example_times = []
prep_times = []
t2 = time.time()
times = []
for scene_id in trange(num_examples):
idx = eval_dataset.scene_id_and_step_to_idx(scene_id, lc_horizon)
torch.cuda.synchronize()
b_ex_time = time.time()
example = eval_dataset[idx]
example = merge_second_batch([example])
example = example_convert_to_torch(example, float_dtype)
with torch.no_grad():
detections = net(example)
torch.cuda.synchronize()
e_ex_time = time.time()
del example, detections
times.append(e_ex_time - b_ex_time)
times = np.array(times)
mean = times.mean()
interval = 1.96 * times.std() / np.sqrt(
len(times)) # 95% confidence interval
return mean, interval
def build_net_v2(ckpt_path, model_cfg, device):
# returns the targer assigner, voxel generator
net = build_network(model_cfg).to(device).eval()
net.load_state_dict(torch.load(ckpt_path))
return net.target_assigner, net.voxel_generator, device, net
def detect(scene_token, config_path, ckpt_path, info_path, root_path,
result_path):
### Read Config file
torch.set_num_threads(2)
#config_path = "configs/nuscenes/all.pp.lowa_large_range_v2.config"
config = pipeline_pb2.TrainEvalPipelineConfig()
with open(config_path, "r") as f:
proto_str = f.read()
text_format.Merge(proto_str, config)
input_cfg = config.eval_input_reader
model_cfg = config.model.second
# config_tool.change_detection_range_v2(model_cfg, [-50, -50, 50, 50])
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
### Build Network, Target Assigner and Voxel Generator
#info_path = '/home/itiv/Desktop/lyft-dataset/infos_val.pkl'
#root_path = '/home/itiv/Desktop/lyft-dataset'
with open(info_path, 'rb') as f:
infos = pickle.load(f)
token2info = {}
for info in infos['infos']:
token2info[info['token']] = info
#ckpt_path = "/home/itiv/Desktop/repo/scenarios_in_CarMaker/BA_Daniel/Lyft-Detector/second.pytorch/second/model/model_large_range_v2/voxelnet-33445.tckpt"
net = build_network(config.model.second).to(device).float().eval()
net.load_state_dict(torch.load(ckpt_path))
eval_input_cfg = config.eval_input_reader
eval_input_cfg.dataset.kitti_root_path = root_path
eval_input_cfg.dataset.kitti_info_path = info_path
dataset = input_reader_builder.build(
eval_input_cfg,
config.model.second,
training=False,
voxel_generator=net.voxel_generator,
target_assigner=net.target_assigner) #.dataset
batch_size = 2
num_workers = 2
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=batch_size,
shuffle=False,
num_workers=num_workers,
pin_memory=False,
collate_fn=merge_second_batch)
target_assigner = net.target_assigner
voxel_generator = net.voxel_generator
classes = target_assigner.classes
detections = []
#tk0 = prog_bar(dataloader, total=len(dataloader))
tk0 = (dataloader)
for idx, examples in enumerate(tk0):
#print(idx)
#print(examples)
try:
example_torch = example_convert_to_torch(examples, device=device)
detections += net(example_torch)
except Exception as e:
print(e)
import pdb
pdb.set_trace()
threshold = 0.2
first_sample_token = detections[0]['metadata']['token']
dict_detections = {"results": {}}
for idx, pred in enumerate((detections)):
pred = thresholded_pred(pred, threshold)
#token = tokens[idx]['token']
token = pred['metadata']['token']
dict_detections['results'].update(
get_pred_dict(pred, token, classes, token2info))
#pred_str = get_pred_str(pred, token)
#predStrings.append(pred_str)
#index = df[df['Id'] == token].index[0]
#df.loc[index, 'PredictionString'] = pred_str
#df.to_csv(f'final.csv', index=False)
#print(dict_detections)
#path_to_result = f'/home/itiv/Desktop/lyft-dataset/detections-largev2.json'
with open(result_path + '/detections_' + scene_token + '.json', 'w') as fp:
json.dump(dict_detections, fp)
