def evaluation(self, detections, label_dir, output_dir):
tags = [itm["tag"] for itm in self._kitti_infos]
calibs = [itm["calib"] for itm in self._kitti_infos]
det_path = os.path.join(output_dir, "data")
assert len(tags) == len(detections) == len(calibs)
self.save_detections(detections, tags, calibs, det_path)
assert len(detections) > 50
dt_annos = kitti.get_label_annos(det_path)
gt_path = os.path.join(label_dir)
val_image_ids = os.listdir(det_path)
val_image_ids = [int(itm.split(".")[0]) for itm in val_image_ids]
val_image_ids.sort()
gt_annos = kitti.get_label_annos(gt_path, val_image_ids)
cls_to_idx = {
"Car": 0,
"Pedestrian": 1,
"Cyclist": 2,
"Van": 3,
"Truck": 4,
"Tram": 5,
"Person_sitting": 6
}
current_classes = [cls_to_idx[itm] for itm in self._class_names]
val_ap_dict = get_official_eval_result(gt_annos, dt_annos,
current_classes)
return val_ap_dict
def evaluation(self, detections, output_dir):
"""
detection
When you want to eval your own dataset, you MUST set correct
the z axis and box z center.
If you want to eval by my KITTI eval function, you must
provide the correct format annotations.
ground_truth_annotations format:
{
bbox: [N, 4], if you fill fake data, MUST HAVE >25 HEIGHT!!!!!!
alpha: [N], you can use -10 to ignore it.
occluded: [N], you can use zero.
truncated: [N], you can use zero.
name: [N]
location: [N, 3] center of 3d box.
dimensions: [N, 3] dim of 3d box.
rotation_y: [N] angle.
}
all fields must be filled, but some fields can fill
zero.
"""
if "annos" not in self._vkitti_infos[0]:
return None
gt_annos = [info["annos"] for info in self._vkitti_infos]
dt_annos = self.convert_detection_to_kitti_annos(detections)
# firstly convert standard detection to kitti-format dt annos
z_axis = 1 # KITTI camera format use y as regular "z" axis.
z_center = 1.0 # KITTI camera box's center is [0.5, 1, 0.5]
# for regular raw lidar data, z_axis = 2, z_center = 0.5.
result_official_dict = get_official_eval_result(gt_annos,
dt_annos,
self._class_names,
z_axis=z_axis,
z_center=z_center)
# SID: Removing COCO evaluation.
# result_coco = get_coco_eval_result(
# gt_annos,
# dt_annos,
# self._class_names,
# z_axis=z_axis,
# z_center=z_center)
return {
"results": {
"official": result_official_dict["result"],
# "coco": result_coco["result"],
},
"detail": {
"eval.kitti": {
"official": result_official_dict["detail"],
# "coco": result_coco["detail"]
}
},
}
def evaluation(self, dt_annos):
"""dt_annos have same format as ground_truth_annotations.
When you want to eval your own dataset, you MUST set correct
the z axis and box z center.
"""
gt_annos = self.ground_truth_annotations
if gt_annos is None:
return None, None
z_axis = 1 # KITTI camera format use y as regular "z" axis.
z_center = 1.0 # KITTI camera box's center is [0.5, 1, 0.5]
# for regular raw lidar data, z_axis = 2, z_center = 0.5.
result_official = get_official_eval_result(gt_annos,
dt_annos,
self._class_names,
z_axis=z_axis,
z_center=z_center)
result_coco = get_coco_eval_result(gt_annos,
dt_annos,
self._class_names,
z_axis=z_axis,
z_center=z_center)
return result_official, result_coco
def evaluation(self, detections, output_dir):
"""
detection
When you want to eval your own dataset, you MUST set correct
the z axis and box z center.
"""
gt_annos = self.ground_truth_annotations
if gt_annos is None:
return None
dt_annos = self.convert_detection_to_kitti_annos(detections)
# firstly convert standard detection to kitti-format dt annos
z_axis = 1 # KITTI camera format use y as regular "z" axis.
z_center = 1.0 # KITTI camera box's center is [0.5, 1, 0.5]
# for regular raw lidar data, z_axis = 2, z_center = 0.5.
result_official_dict = get_official_eval_result(gt_annos,
dt_annos,
self._class_names,
z_axis=z_axis,
z_center=z_center)
result_coco = get_coco_eval_result(gt_annos,
dt_annos,
self._class_names,
z_axis=z_axis,
z_center=z_center)
return {
"results": {
"official": result_official_dict["result"],
"coco": result_coco["result"],
},
"detail": {
"eval.kitti": {
"official": result_official_dict["detail"],
"coco": result_coco["detail"]
}
},
}
def evaluation_kitti(self, detections, output_dir):
"""eval by kitti evaluation tool.
I use num_lidar_pts to set easy, mod, hard.
easy: num>15, mod: num>7, hard: num>0.
"""
print("++++++++NuScenes KITTI unofficial Evaluation:")
print(
"++++++++easy: num_lidar_pts>15, mod: num_lidar_pts>7, hard: num_lidar_pts>0"
)
print("++++++++The bbox AP is invalid. Don't forget to ignore it.")
class_names = self._class_names
gt_annos = self.ground_truth_annotations
if gt_annos is None:
return None
gt_annos = deepcopy(gt_annos)
detections = deepcopy(detections)
dt_annos = []
for det in detections:
final_box_preds = det["box3d_lidar"].detach().cpu().numpy()
label_preds = det["label_preds"].detach().cpu().numpy()
scores = det["scores"].detach().cpu().numpy()
anno = kitti.get_start_result_anno()
num_example = 0
box3d_lidar = final_box_preds
for j in range(box3d_lidar.shape[0]):
anno["bbox"].append(np.array([0, 0, 50, 50]))
anno["alpha"].append(-10)
anno["dimensions"].append(box3d_lidar[j, 3:6])
anno["location"].append(box3d_lidar[j, :3])
anno["rotation_y"].append(box3d_lidar[j, 6])
anno["name"].append(class_names[int(label_preds[j])])
anno["truncated"].append(0.0)
anno["occluded"].append(0)
anno["score"].append(scores[j])
num_example += 1
if num_example != 0:
anno = {n: np.stack(v) for n, v in anno.items()}
dt_annos.append(anno)
else:
dt_annos.append(kitti.empty_result_anno())
num_example = dt_annos[-1]["name"].shape[0]
dt_annos[-1]["metadata"] = det["metadata"]
for anno in gt_annos:
names = anno["name"].tolist()
mapped_names = []
for n in names:
if n in self.NameMapping:
mapped_names.append(self.NameMapping[n])
else:
mapped_names.append(n)
anno["name"] = np.array(mapped_names)
for anno in dt_annos:
names = anno["name"].tolist()
mapped_names = []
for n in names:
if n in self.NameMapping:
mapped_names.append(self.NameMapping[n])
else:
mapped_names.append(n)
anno["name"] = np.array(mapped_names)
mapped_class_names = []
for n in self._class_names:
if n in self.NameMapping:
mapped_class_names.append(self.NameMapping[n])
else:
mapped_class_names.append(n)
z_axis = 2
z_center = 0.5
# for regular raw lidar data, z_axis = 2, z_center = 0.5.
result_official_dict = get_official_eval_result(gt_annos,
dt_annos,
mapped_class_names,
z_axis=z_axis,
z_center=z_center)
result_coco = get_coco_eval_result(gt_annos,
dt_annos,
mapped_class_names,
z_axis=z_axis,
z_center=z_center)
return {
"results": {
"official": result_official_dict["result"],
"coco": result_coco["result"],
},
"detail": {
"official": result_official_dict["detail"],
"coco": result_coco["detail"],
},
}
def train(config_path,
model_dir,
result_path=None,
create_folder=False,
display_step=50,
summary_step=5,
pickle_result=True):
"""train a VoxelNet model specified by a config file.
"""
if create_folder:
if pathlib.Path(model_dir).exists():
model_dir = torchplus.train.create_folder(model_dir)
model_dir = pathlib.Path(model_dir)
model_dir.mkdir(parents=True, exist_ok=True)
eval_checkpoint_dir = model_dir / 'eval_checkpoints'
eval_checkpoint_dir.mkdir(parents=True, exist_ok=True)
if result_path is None:
result_path = model_dir / 'results'
config_file_bkp = "pipeline.config"
config = pipeline_pb2.TrainEvalPipelineConfig()
with open(config_path, "r") as f:
proto_str = f.read()
text_format.Merge(proto_str, config)
shutil.copyfile(config_path, str(model_dir / config_file_bkp))
input_cfg = config.train_input_reader
eval_input_cfg = config.eval_input_reader
model_cfg = config.model.second
train_cfg = config.train_config
class_names = list(input_cfg.class_names)
######################
# BUILD VOXEL GENERATOR
######################
voxel_generator = voxel_builder.build(model_cfg.voxel_generator)
######################
# BUILD TARGET ASSIGNER
######################
bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]]
box_coder = box_coder_builder.build(model_cfg.box_coder)
target_assigner_cfg = model_cfg.target_assigner
target_assigner = target_assigner_builder.build(target_assigner_cfg,
bv_range, box_coder)
######################
# BUILD NET
######################
center_limit_range = model_cfg.post_center_limit_range
net = second_builder.build(model_cfg, voxel_generator, target_assigner)
net.cuda()
# net_train = torch.nn.DataParallel(net).cuda()
print("num_trainable parameters:", len(list(net.parameters())))
# for n, p in net.named_parameters():
# print(n, p.shape)
######################
# BUILD OPTIMIZER
######################
# we need global_step to create lr_scheduler, so restore net first.
torchplus.train.try_restore_latest_checkpoints(model_dir, [net])
gstep = net.get_global_step() - 1
optimizer_cfg = train_cfg.optimizer
if train_cfg.enable_mixed_precision:
net.half()
net.metrics_to_float()
net.convert_norm_to_float(net)
optimizer = optimizer_builder.build(optimizer_cfg, net.parameters())
if train_cfg.enable_mixed_precision:
loss_scale = train_cfg.loss_scale_factor
mixed_optimizer = torchplus.train.MixedPrecisionWrapper(
optimizer, loss_scale)
else:
mixed_optimizer = optimizer
# must restore optimizer AFTER using MixedPrecisionWrapper
torchplus.train.try_restore_latest_checkpoints(model_dir,
[mixed_optimizer])
lr_scheduler = lr_scheduler_builder.build(optimizer_cfg, optimizer, gstep)
if train_cfg.enable_mixed_precision:
float_dtype = torch.float16
else:
float_dtype = torch.float32
######################
# PREPARE INPUT
######################
dataset = input_reader_builder.build(input_cfg,
model_cfg,
training=True,
voxel_generator=voxel_generator,
target_assigner=target_assigner)
eval_dataset = input_reader_builder.build(eval_input_cfg,
model_cfg,
training=False,
voxel_generator=voxel_generator,
target_assigner=target_assigner)
def _worker_init_fn(worker_id):
time_seed = np.array(time.time(), dtype=np.int32)
np.random.seed(time_seed + worker_id)
print(f"WORKER {worker_id} seed:", np.random.get_state()[1][0])
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=input_cfg.batch_size,
shuffle=True,
num_workers=input_cfg.num_workers,
pin_memory=False,
collate_fn=merge_second_batch,
worker_init_fn=_worker_init_fn)
eval_dataloader = torch.utils.data.DataLoader(
eval_dataset,
batch_size=eval_input_cfg.batch_size,
shuffle=False,
num_workers=eval_input_cfg.num_workers,
pin_memory=False,
collate_fn=merge_second_batch)
data_iter = iter(dataloader)
######################
# TRAINING
######################
log_path = model_dir / 'log.txt'
logf = open(log_path, 'a')
logf.write(proto_str)
logf.write("\n")
summary_dir = model_dir / 'summary'
summary_dir.mkdir(parents=True, exist_ok=True)
writer = SummaryWriter(str(summary_dir))
total_step_elapsed = 0
remain_steps = train_cfg.steps - net.get_global_step()
t = time.time()
ckpt_start_time = t
total_loop = train_cfg.steps // train_cfg.steps_per_eval + 1
# total_loop = remain_steps // train_cfg.steps_per_eval + 1
clear_metrics_every_epoch = train_cfg.clear_metrics_every_epoch
if train_cfg.steps % train_cfg.steps_per_eval == 0:
total_loop -= 1
mixed_optimizer.zero_grad()
try:
for _ in range(total_loop):
if total_step_elapsed + train_cfg.steps_per_eval > train_cfg.steps:
steps = train_cfg.steps % train_cfg.steps_per_eval
else:
steps = train_cfg.steps_per_eval
for step in range(steps):
lr_scheduler.step()
try:
example = next(data_iter)
except StopIteration:
print("end epoch")
if clear_metrics_every_epoch:
net.clear_metrics()
data_iter = iter(dataloader)
example = next(data_iter)
example_torch = example_convert_to_torch(example, float_dtype)
batch_size = example["anchors"].shape[0]
ret_dict = net(example_torch)
# box_preds = ret_dict["box_preds"]
cls_preds = ret_dict["cls_preds"]
loss = ret_dict["loss"].mean()
cls_loss_reduced = ret_dict["cls_loss_reduced"].mean()
loc_loss_reduced = ret_dict["loc_loss_reduced"].mean()
cls_pos_loss = ret_dict["cls_pos_loss"]
cls_neg_loss = ret_dict["cls_neg_loss"]
loc_loss = ret_dict["loc_loss"]
cls_loss = ret_dict["cls_loss"]
dir_loss_reduced = ret_dict["dir_loss_reduced"]
cared = ret_dict["cared"]
labels = example_torch["labels"]
if train_cfg.enable_mixed_precision:
loss *= loss_scale
loss.backward()
torch.nn.utils.clip_grad_norm_(net.parameters(), 10.0)
mixed_optimizer.step()
mixed_optimizer.zero_grad()
net.update_global_step()
net_metrics = net.update_metrics(cls_loss_reduced,
loc_loss_reduced, cls_preds,
labels, cared)
step_time = (time.time() - t)
t = time.time()
metrics = {}
num_pos = int((labels > 0)[0].float().sum().cpu().numpy())
num_neg = int((labels == 0)[0].float().sum().cpu().numpy())
if 'anchors_mask' not in example_torch:
num_anchors = example_torch['anchors'].shape[1]
else:
num_anchors = int(example_torch['anchors_mask'][0].sum())
global_step = net.get_global_step()
if global_step % display_step == 0:
loc_loss_elem = [
float(loc_loss[:, :, i].sum().detach().cpu().numpy() /
batch_size) for i in range(loc_loss.shape[-1])
]
metrics["step"] = global_step
metrics["steptime"] = step_time
metrics.update(net_metrics)
metrics["loss"] = {}
metrics["loss"]["loc_elem"] = loc_loss_elem
metrics["loss"]["cls_pos_rt"] = float(
cls_pos_loss.detach().cpu().numpy())
metrics["loss"]["cls_neg_rt"] = float(
cls_neg_loss.detach().cpu().numpy())
# if unlabeled_training:
# metrics["loss"]["diff_rt"] = float(
# diff_loc_loss_reduced.detach().cpu().numpy())
if model_cfg.use_direction_classifier:
metrics["loss"]["dir_rt"] = float(
dir_loss_reduced.detach().cpu().numpy())
metrics["num_vox"] = int(example_torch["voxels"].shape[0])
metrics["num_pos"] = int(num_pos)
metrics["num_neg"] = int(num_neg)
metrics["num_anchors"] = int(num_anchors)
metrics["lr"] = float(
mixed_optimizer.param_groups[0]['lr'])
metrics["image_idx"] = example['image_idx'][0]
flatted_metrics = flat_nested_json_dict(metrics)
flatted_summarys = flat_nested_json_dict(metrics, "/")
for k, v in flatted_summarys.items():
if isinstance(v, (list, tuple)):
v = {str(i): e for i, e in enumerate(v)}
writer.add_scalars(k, v, global_step)
else:
writer.add_scalar(k, v, global_step)
metrics_str_list = []
for k, v in flatted_metrics.items():
if isinstance(v, float):
metrics_str_list.append(f"{k}={v:.3}")
elif isinstance(v, (list, tuple)):
if v and isinstance(v[0], float):
v_str = ', '.join([f"{e:.3}" for e in v])
metrics_str_list.append(f"{k}=[{v_str}]")
else:
metrics_str_list.append(f"{k}={v}")
else:
metrics_str_list.append(f"{k}={v}")
log_str = ', '.join(metrics_str_list)
print(log_str, file=logf)
print(log_str)
ckpt_elasped_time = time.time() - ckpt_start_time
if ckpt_elasped_time > train_cfg.save_checkpoints_secs:
torchplus.train.save_models(model_dir, [net, optimizer],
net.get_global_step())
ckpt_start_time = time.time()
total_step_elapsed += steps
torchplus.train.save_models(model_dir, [net, optimizer],
net.get_global_step())
# Ensure that all evaluation points are saved forever
torchplus.train.save_models(eval_checkpoint_dir, [net, optimizer],
net.get_global_step(),
max_to_keep=100)
net.eval()
result_path_step = result_path / f"step_{net.get_global_step()}"
result_path_step.mkdir(parents=True, exist_ok=True)
print("#################################")
print("#################################", file=logf)
print("# EVAL")
print("# EVAL", file=logf)
print("#################################")
print("#################################", file=logf)
print("Generate output labels...")
print("Generate output labels...", file=logf)
t = time.time()
dt_annos = []
prog_bar = ProgressBar()
prog_bar.start(len(eval_dataset) // eval_input_cfg.batch_size + 1)
for example in iter(eval_dataloader):
example = example_convert_to_torch(example, float_dtype)
if pickle_result:
dt_annos += predict_kitti_to_anno(net, example,
class_names,
center_limit_range,
model_cfg.lidar_input)
else:
_predict_kitti_to_file(net, example, result_path_step,
class_names, center_limit_range,
model_cfg.lidar_input)
prog_bar.print_bar()
sec_per_ex = len(eval_dataset) / (time.time() - t)
print(f"avg forward time per example: {net.avg_forward_time:.3f}")
print(
f"avg postprocess time per example: {net.avg_postprocess_time:.3f}"
)
net.clear_time_metrics()
print(f'generate label finished({sec_per_ex:.2f}/s). start eval:')
print(f'generate label finished({sec_per_ex:.2f}/s). start eval:',
file=logf)
gt_annos = [
info["annos"] for info in eval_dataset.dataset.kitti_infos
]
if not pickle_result:
dt_annos = kitti.get_label_annos(result_path_step)
result, mAPbbox, mAPbev, mAP3d, mAPaos = get_official_eval_result(
gt_annos, dt_annos, class_names, return_data=True)
print(result, file=logf)
print(result)
writer.add_text('eval_result', result, global_step)
for i, class_name in enumerate(class_names):
writer.add_scalar('bev_ap:{}'.format(class_name),
mAPbev[i, 1, 0], global_step)
writer.add_scalar('3d_ap:{}'.format(class_name),
mAP3d[i, 1, 0], global_step)
writer.add_scalar('aos_ap:{}'.format(class_name),
mAPaos[i, 1, 0], global_step)
writer.add_scalar('bev_map', np.mean(mAPbev[:, 1, 0]), global_step)
writer.add_scalar('3d_map', np.mean(mAP3d[:, 1, 0]), global_step)
writer.add_scalar('aos_map', np.mean(mAPaos[:, 1, 0]), global_step)
result = get_coco_eval_result(gt_annos, dt_annos, class_names)
print(result, file=logf)
print(result)
if pickle_result:
with open(result_path_step / "result.pkl", 'wb') as f:
pickle.dump(dt_annos, f)
writer.add_text('eval_result', result, global_step)
net.train()
except Exception as e:
torchplus.train.save_models(model_dir, [net, optimizer],
net.get_global_step())
logf.close()
raise e
# save model before exit
torchplus.train.save_models(model_dir, [net, optimizer],
net.get_global_step())
logf.close()
def evaluate(config_path,
model_dir,
result_path=None,
predict_test=False,
ckpt_path=None,
ref_detfile=None,
pickle_result=True):
model_dir = pathlib.Path(model_dir)
if predict_test:
result_name = 'predict_test'
else:
result_name = 'eval_results'
if result_path is None:
result_path = model_dir / result_name
else:
result_path = pathlib.Path(result_path)
config = pipeline_pb2.TrainEvalPipelineConfig()
with open(config_path, "r") as f:
proto_str = f.read()
text_format.Merge(proto_str, config)
input_cfg = config.eval_input_reader
model_cfg = config.model.second
train_cfg = config.train_config
class_names = list(input_cfg.class_names)
center_limit_range = model_cfg.post_center_limit_range
######################
# BUILD VOXEL GENERATOR
######################
voxel_generator = voxel_builder.build(model_cfg.voxel_generator)
bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]]
box_coder = box_coder_builder.build(model_cfg.box_coder)
target_assigner_cfg = model_cfg.target_assigner
target_assigner = target_assigner_builder.build(target_assigner_cfg,
bv_range, box_coder)
net = second_builder.build(model_cfg, voxel_generator, target_assigner)
net.cuda()
if train_cfg.enable_mixed_precision:
net.half()
net.metrics_to_float()
net.convert_norm_to_float(net)
if ckpt_path is None:
torchplus.train.try_restore_latest_checkpoints(model_dir, [net])
else:
torchplus.train.restore(ckpt_path, net)
eval_dataset = input_reader_builder.build(input_cfg,
model_cfg,
training=False,
voxel_generator=voxel_generator,
target_assigner=target_assigner)
eval_dataloader = torch.utils.data.DataLoader(
eval_dataset,
batch_size=input_cfg.batch_size,
shuffle=False,
num_workers=input_cfg.num_workers,
pin_memory=False,
collate_fn=merge_second_batch)
if train_cfg.enable_mixed_precision:
float_dtype = torch.float16
else:
float_dtype = torch.float32
net.eval()
result_path_step = result_path / f"step_{net.get_global_step()}"
result_path_step.mkdir(parents=True, exist_ok=True)
t = time.time()
dt_annos = []
global_set = None
print("Generate output labels...")
bar = ProgressBar()
bar.start(len(eval_dataset) // input_cfg.batch_size + 1)
for example in iter(eval_dataloader):
example = example_convert_to_torch(example, float_dtype)
if pickle_result:
dt_annos += predict_kitti_to_anno(net, example, class_names,
center_limit_range,
model_cfg.lidar_input,
global_set)
else:
_predict_kitti_to_file(net, example, result_path_step, class_names,
center_limit_range, model_cfg.lidar_input)
bar.print_bar()
sec_per_example = len(eval_dataset) / (time.time() - t)
print(f'generate label finished({sec_per_example:.2f}/s). start eval:')
print(f"avg forward time per example: {net.avg_forward_time:.3f}")
print(f"avg postprocess time per example: {net.avg_postprocess_time:.3f}")
if not predict_test:
gt_annos = [info["annos"] for info in eval_dataset.dataset.kitti_infos]
if not pickle_result:
dt_annos = kitti.get_label_annos(result_path_step)
result = get_official_eval_result(gt_annos, dt_annos, class_names)
print(result)
result = get_coco_eval_result(gt_annos, dt_annos, class_names)
print(result)
if pickle_result:
with open(result_path_step / "result.pkl", 'wb') as f:
pickle.dump(dt_annos, f)
def evaluate(config_path,
model_dir,
use_second_stage=False,
use_endtoend=False,
result_path=None,
predict_test=False,
ckpt_path=None,
ref_detfile=None,
pickle_result=True,
measure_time=False,
batch_size=None):
model_dir = pathlib.Path(model_dir)
if predict_test:
result_name = 'predict_test_0095'
else:
result_name = 'eval_results'
if result_path is None:
result_path = model_dir / result_name
else:
result_path = pathlib.Path(result_path)
config = pipeline_pb2.TrainEvalPipelineConfig()
with open(config_path, "r") as f:
proto_str = f.read()
text_format.Merge(proto_str, config)
input_cfg = config.eval_input_reader
model_cfg = config.model.second
train_cfg = config.train_config
center_limit_range = model_cfg.post_center_limit_range
######################
# BUILD VOXEL GENERATOR
######################
voxel_generator = voxel_builder.build(model_cfg.voxel_generator)
bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]]
box_coder = box_coder_builder.build(model_cfg.box_coder)
target_assigner_cfg = model_cfg.target_assigner
target_assigner = target_assigner_builder.build(target_assigner_cfg,
bv_range, box_coder)
class_names = target_assigner.classes
if use_second_stage:
net = second_2stage_builder.build(model_cfg, voxel_generator, target_assigner, measure_time=measure_time)
elif use_endtoend:
net = second_endtoend_builder.build(model_cfg, voxel_generator, target_assigner, measure_time=measure_time)
else:
net = second_builder.build(model_cfg, voxel_generator, target_assigner, measure_time=measure_time)
net.cuda()
#########################################
# net = torch.nn.DataParallel(net)
#########################################
if ckpt_path is None:
torchplus.train.try_restore_latest_checkpoints(model_dir, [net])
else:
torchplus.train.restore(ckpt_path, net)
if train_cfg.enable_mixed_precision:
net.half()
print("half inference!")
net.metrics_to_float()
net.convert_norm_to_float(net)
batch_size = batch_size or input_cfg.batch_size
eval_dataset = input_reader_builder_tr.build(
input_cfg,
model_cfg,
training=False,
voxel_generator=voxel_generator,
target_assigner=target_assigner)
eval_dataloader = torch.utils.data.DataLoader(
eval_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=0,# input_cfg.num_workers,
pin_memory=False,
collate_fn=merge_second_batch)
if train_cfg.enable_mixed_precision:
float_dtype = torch.float16
else:
float_dtype = torch.float32
net.eval()
result_path_step = result_path / f"step_{net.get_global_step()}"
result_path_step.mkdir(parents=True, exist_ok=True)
t = time.time()
dt_annos = []
global_set = None
print("Generate output labels...")
bar = ProgressBar()
bar.start((len(eval_dataset) + batch_size - 1) // batch_size)
prep_example_times = []
prep_times = []
t2 = time.time()
for example in iter(eval_dataloader):
if measure_time:
prep_times.append(time.time() - t2)
t1 = time.time()
torch.cuda.synchronize()
example = example_convert_to_torch(example, float_dtype)
if measure_time:
torch.cuda.synchronize()
prep_example_times.append(time.time() - t1)
if pickle_result:
dt_annos += predict_kitti_to_anno(
net, example, class_names, center_limit_range,
model_cfg.lidar_input, global_set)
else:
_predict_kitti_to_file(net, example, result_path_step, class_names,
center_limit_range, model_cfg.lidar_input)
# print(json.dumps(net.middle_feature_extractor.middle_conv.sparity_dict))
bar.print_bar()
if measure_time:
t2 = time.time()
sec_per_example = len(eval_dataset) / (time.time() - t)
print(f'generate label finished({sec_per_example:.2f}/s). start eval:')
if measure_time:
print(f"avg example to torch time: {np.mean(prep_example_times) * 1000:.3f} ms")
print(f"avg prep time: {np.mean(prep_times) * 1000:.3f} ms")
for name, val in net.get_avg_time_dict().items():
print(f"avg {name} time = {val * 1000:.3f} ms")
if not predict_test:
gt_annos = [info["annos"] for info in eval_dataset.dataset.kitti_infos]
img_idx = [info["image_idx"] for info in eval_dataset.dataset.kitti_infos]
if not pickle_result:
dt_annos = kitti.get_label_annos(result_path_step)
result = get_official_eval_result(gt_annos, dt_annos, class_names)
# print(json.dumps(result, indent=2))
print(result)
result = get_coco_eval_result(gt_annos, dt_annos, class_names)
print(result)
if pickle_result:
with open(result_path_step / "result.pkl", 'wb') as f:
pickle.dump(dt_annos, f)
# annos to txt file
if True:
os.makedirs(str(result_path_step) + '/txt', exist_ok=True)
for i in range(len(dt_annos)):
dt_annos[i]['dimensions'] = dt_annos[i]['dimensions'][:, [1, 2, 0]]
result_lines = kitti.annos_to_kitti_label(dt_annos[i])
image_idx = img_idx[i]
with open(str(result_path_step) + '/txt/%06d.txt' % image_idx, 'w') as f:
for result_line in result_lines:
f.write(result_line + '\n')
abcd = 1
else:
os.makedirs(str(result_path_step) + '/txt', exist_ok=True)
img_idx = [info["image_idx"] for info in eval_dataset.dataset.kitti_infos]
for i in range(len(dt_annos)):
dt_annos[i]['dimensions'] = dt_annos[i]['dimensions'][:, [1, 2, 0]]
result_lines = kitti.annos_to_kitti_label(dt_annos[i])
image_idx = img_idx[i]
with open(str(result_path_step) + '/txt/%06d.txt' % image_idx, 'w') as f:
for result_line in result_lines:
f.write(result_line + '\n')
def train(config_path,
model_dir,
use_fusion=True,
use_ft=False,
use_second_stage=True,
use_endtoend=True,
result_path=None,
create_folder=False,
display_step=50,
summary_step=5,
local_rank=0,
pickle_result=True,
patchs=None):
"""train a VoxelNet mod[el specified by a config file.
"""
############ tracking
config_tr_path = '/mnt/new_iou/second.pytorch/second/mmMOT/experiments/second/spatio_test/config.yaml'
load_tr_path = '/mnt/new_iou/second.pytorch/second/mmMOT/experiments/second/spatio_test/results'
with open(config_tr_path) as f:
config_tr = yaml.load(f, Loader=yaml.FullLoader)
result_path_tr = load_tr_path
config_tr = EasyDict(config_tr['common'])
config_tr.save_path = os.path.dirname(config_tr_path)
# create model
# model_tr = build_model(config_tr)
# model_tr.cuda()
# optimizer_tr = build_optim(model_tr, config_tr)
criterion_tr = build_criterion(config_tr.loss)
last_iter = -1
best_mota = 0
# if load_tr_path:
# if False:
# best_mota, last_iter = load_state(
# load_tr_path, model_tr, optimizer=optimizer_tr)
# else:
# load_state(load_tr_path, model_tr)
cudnn.benchmark = True
# Data loading code
train_transform, valid_transform = build_augmentation(config_tr.augmentation)
# # train
# train_dataset = build_dataset(
# config_tr,
# set_source='train',
# evaluate=False,
# train_transform=train_transform)
# trainval_dataset = build_dataset(
# config_tr,
# set_source='train',
# evaluate=True,
# valid_transform=valid_transform)
# val_dataset = build_dataset(
# config_tr,
# set_source='val',
# evaluate=True,
# valid_transform=valid_transform)
# train_sampler = DistributedGivenIterationSampler(
# train_dataset,
# config_tr.lr_scheduler.max_iter,
# config_tr.batch_size,
# world_size=1,
# rank=0,
# last_iter=last_iter)
# import pdb; pdb.set_trace()
# train_loader = DataLoader(
# train_dataset,
# batch_size=config_tr.batch_size,
# shuffle=False,
# num_workers=config_tr.workers,
# pin_memory=True)
tb_logger = SummaryWriter(config_tr.save_path + '/events')
logger = create_logger('global_logger', config_tr.save_path + '/log.txt')
# logger.info('args: {}'.format(pprint.pformat(args)))
logger.info('config: {}'.format(pprint.pformat(config_tr)))
# tracking_module = TrackingModule(model_tr, criterion_tr,
# config_tr.det_type)
# tracking_module.model.train()
#### tracking setup done
if create_folder:
if pathlib.Path(model_dir).exists():
model_dir = torchplus.train.create_folder(model_dir)
patchs = patchs or []
model_dir = pathlib.Path(model_dir)
model_dir.mkdir(parents=True, exist_ok=True)
if result_path is None:
result_path = model_dir / 'results'
config_file_bkp = "pipeline.config"
config = pipeline_pb2.TrainEvalPipelineConfig()
with open(config_path, "r") as f:
proto_str = f.read()
text_format.Merge(proto_str, config)
for patch in patchs:
patch = "config." + patch
exec(patch)
shutil.copyfile(config_path, str(model_dir / config_file_bkp))
input_cfg = config.train_input_reader
eval_input_cfg = config.eval_input_reader
model_cfg = config.model.second
train_cfg = config.train_config
######################
# BUILD VOXEL GENERATOR
######################
voxel_generator = voxel_builder.build(model_cfg.voxel_generator)
######################
# BUILD TARGET ASSIGNER
######################
bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]]
box_coder = box_coder_builder.build(model_cfg.box_coder)
target_assigner_cfg = model_cfg.target_assigner
target_assigner = target_assigner_builder.build(target_assigner_cfg,
bv_range, box_coder)
class_names = target_assigner.classes
######################
# BUILD NET
######################
center_limit_range = model_cfg.post_center_limit_range
# if use_second_stage:
# net = second_2stage_builder.build(model_cfg, voxel_generator, target_assigner)
if use_endtoend:
net = second_endtoend_builder_spatio.build(model_cfg, voxel_generator, target_assigner, criterion_tr, config_tr.det_type)
else:
net = second_builder.build(model_cfg, voxel_generator, target_assigner)
net.cuda()
print("num_trainable parameters:", len(list(net.parameters())))
for n, p in net.named_parameters():
print(n, p.shape)
# pth_name = './pre_weight/first_stage_gating_det/voxelnet-17013.tckpt'
pth_name = './pre_weight/second_stage_gating_det/voxelnet-35000.tckpt'
res_pre_weights = torch.load(pth_name)
new_res_state_dict = OrderedDict()
model_dict = net.state_dict()
for k,v in res_pre_weights.items():
if 'global_step' not in k:
# if 'dir' not in k:
new_res_state_dict[k] = v
model_dict.update(new_res_state_dict)
net.load_state_dict(model_dict)
# for k, weight in dict(net.named_parameters()).items(): # lidar_conv, p_lidar_conv, fusion_module, w_det, w_link, appearance, point_net
# if 'middle_feature_extractor' in '%s'%(k) or 'rpn' in '%s'%(k) or 'second_rpn' in '%s'%(k):
# weight.requires_grad = False
# BUILD OPTIMIZER
#####################
# we need global_step to create lr_scheduler, so restore net first.
torchplus.train.try_restore_latest_checkpoints(model_dir, [net])
gstep = net.get_global_step() - 1
optimizer_cfg = train_cfg.optimizer
if train_cfg.enable_mixed_precision:
net.half()
net.metrics_to_float()
net.convert_norm_to_float(net)
loss_scale = train_cfg.loss_scale_factor
mixed_optimizer = optimizer_builder.build(optimizer_cfg, net, mixed=train_cfg.enable_mixed_precision, loss_scale=loss_scale)
optimizer = mixed_optimizer
# must restore optimizer AFTER using MixedPrecisionWrapper
torchplus.train.try_restore_latest_checkpoints(model_dir,
[mixed_optimizer])
lr_scheduler = lr_scheduler_builder.build(optimizer_cfg, optimizer, train_cfg.steps)
if train_cfg.enable_mixed_precision:
float_dtype = torch.float16
else:
float_dtype = torch.float32
######################
# PREPARE INPUT
######################
# import pdb; pdb.set_trace()
dataset = input_reader_builder_tr_vid_spatio.build(
input_cfg,
model_cfg,
training=True,
voxel_generator=voxel_generator,
target_assigner=target_assigner,
config_tr=config_tr,
set_source='train',
evaluate=False,
train_transform=train_transform)
eval_dataset = input_reader_builder_tr_vid_spatio.build(
eval_input_cfg,
model_cfg,
training=False,
voxel_generator=voxel_generator,
target_assigner=target_assigner,
config_tr=config_tr,
set_source='val',
evaluate=True,
valid_transform=valid_transform)
def _worker_init_fn(worker_id):
time_seed = np.array(time.time(), dtype=np.int32)
np.random.seed(time_seed + worker_id)
print(f"WORKER {worker_id} seed:", np.random.get_state()[1][0])
dataloader = torch.utils.data.DataLoader(
dataset,
batch_size=input_cfg.batch_size,
shuffle=True,
num_workers=input_cfg.num_workers,
pin_memory=False,
collate_fn=merge_second_batch_tr_vid_spatio,
worker_init_fn=_worker_init_fn)
eval_dataloader = torch.utils.data.DataLoader(
eval_dataset,
batch_size=eval_input_cfg.batch_size,
shuffle=False,
num_workers=eval_input_cfg.num_workers,
pin_memory=False,
collate_fn=merge_second_batch_tr_vid_spatio)
data_iter = iter(dataloader)
######################
# TRAINING
######################
training_detail = []
log_path = model_dir / 'log.txt'
training_detail_path = model_dir / 'log.json'
if training_detail_path.exists():
with open(training_detail_path, 'r') as f:
training_detail = json.load(f)
logf = open(log_path, 'a')
logf.write(proto_str)
logf.write("\n")
summary_dir = model_dir / 'summary'
summary_dir.mkdir(parents=True, exist_ok=True)
writer = SummaryWriter(str(summary_dir))
total_step_elapsed = 0
remain_steps = train_cfg.steps - net.get_global_step()
t = time.time()
ckpt_start_time = t
total_loop = train_cfg.steps // train_cfg.steps_per_eval + 1
clear_metrics_every_epoch = train_cfg.clear_metrics_every_epoch
if train_cfg.steps % train_cfg.steps_per_eval == 0:
total_loop -= 1
mixed_optimizer.zero_grad()
# optimizer_tr.zero_grad()
logger = logging.getLogger('global_logger')
best_mota = 0
losses = AverageMeter(config_tr.print_freq)
total_steps = train_cfg.steps
total_loop = total_steps // len(dataloader)
kkkk = 0
for step in range(total_loop):
for i, (example) in enumerate(dataloader):
curr_step = 0 + i
kkkk += 1
lr_scheduler.step(net.get_global_step())
example_torch = example_convert_to_torch(example, float_dtype)
batch_size = example["anchors"].shape[0]
ret_dict = net(example_torch, train_param=True)
cls_preds = ret_dict["cls_preds"]
loss = ret_dict["loss"].mean()
cls_loss_reduced = ret_dict["cls_loss_reduced"].mean()
loc_loss_reduced = ret_dict["loc_loss_reduced"].mean()
cls_pos_loss = ret_dict["cls_pos_loss"]
cls_neg_loss = ret_dict["cls_neg_loss"]
loc_loss = ret_dict["loc_loss"]
cls_loss = ret_dict["cls_loss"]
dir_loss_reduced = ret_dict["dir_loss_reduced"]
cared = ret_dict["cared"]
# loss_tr = ret_dict["loss_tr"]
if use_second_stage or use_endtoend:
labels = ret_dict["labels"]
else:
labels = example_torch["labels"]
if train_cfg.enable_mixed_precision:
loss *= loss_scale
try:
loss.backward()
except:
abc = 1
# import pdb; pdb.set_trace()
# abc = 1
# torch.nn.utils.clip_grad_norm_(net.parameters(), 10.0)
# optimizer_tr.step()
# optimizer_tr.zero_grad()
mixed_optimizer.step()
mixed_optimizer.zero_grad()
net.update_global_step()
net_metrics = net.update_metrics(cls_loss_reduced,
loc_loss_reduced, cls_preds,
labels, cared)
step_time = (time.time() - t)
t = time.time()
metrics = {}
num_pos = int((labels > 0)[0].float().sum().cpu().numpy())
num_neg = int((labels == 0)[0].float().sum().cpu().numpy())
if 'anchors_mask' not in example_torch:
num_anchors = example_torch['anchors'].shape[1]
else:
num_anchors = int(example_torch['anchors_mask'][0].sum())
global_step = net.get_global_step()
# print(step)
if global_step % display_step == 0:
loc_loss_elem = [
float(loc_loss[:, :, i].sum().detach().cpu().numpy() /
batch_size) for i in range(loc_loss.shape[-1])
]
metrics["type"] = "step_info"
metrics["step"] = global_step
metrics["steptime"] = step_time
metrics.update(net_metrics)
metrics["loss"] = {}
metrics["loss"]["loc_elem"] = loc_loss_elem
metrics["loss"]["cls_pos_rt"] = float(
cls_pos_loss.detach().cpu().numpy())
metrics["loss"]["cls_neg_rt"] = float(
cls_neg_loss.detach().cpu().numpy())
if model_cfg.use_direction_classifier:
metrics["loss"]["dir_rt"] = float(
dir_loss_reduced.detach().cpu().numpy())
metrics["num_vox"] = int(example_torch["voxels"].shape[0])
metrics["num_pos"] = int(num_pos)
metrics["num_neg"] = int(num_neg)
metrics["num_anchors"] = int(num_anchors)
metrics["lr"] = float(
optimizer.lr)
metrics["image_idx"] = example['image_idx'][0][7:]
training_detail.append(metrics)
flatted_metrics = flat_nested_json_dict(metrics)
flatted_summarys = flat_nested_json_dict(metrics, "/")
for k, v in flatted_summarys.items():
if isinstance(v, (list, tuple)):
v = {str(i): e for i, e in enumerate(v)}
if type(v) != str and ('loc_elem' not in k):
writer.add_scalars(k, v, global_step)
else:
if (type(v) != str) and ('loc_elem' not in k):
writer.add_scalar(k, v, global_step)
metrics_str_list = []
for k, v in flatted_metrics.items():
if isinstance(v, float):
metrics_str_list.append(f"{k}={v:.3}")
elif isinstance(v, (list, tuple)):
if v and isinstance(v[0], float):
v_str = ', '.join([f"{e:.3}" for e in v])
metrics_str_list.append(f"{k}=[{v_str}]")
else:
metrics_str_list.append(f"{k}={v}")
else:
metrics_str_list.append(f"{k}={v}")
log_str = ', '.join(metrics_str_list)
print(log_str, file=logf)
print(log_str)
ckpt_elasped_time = time.time() - ckpt_start_time
if ckpt_elasped_time > train_cfg.save_checkpoints_secs:
torchplus.train.save_models(model_dir, [net, optimizer], net.get_global_step())
ckpt_start_time = time.time()
if kkkk > 0 and (kkkk) % config_tr.val_freq == 0:
# if True:
torchplus.train.save_models(model_dir, [net, optimizer], net.get_global_step())
net.eval()
result_path_step = result_path / f"step_{net.get_global_step()}"
result_path_step.mkdir(parents=True, exist_ok=True)
print("#################################")
print("#################################", file=logf)
print("# EVAL")
print("# EVAL", file=logf)
print("#################################")
print("#################################", file=logf)
print("Generate output labels...")
print("Generate output labels...", file=logf)
t = time.time()
dt_annos = []
prog_bar = ProgressBar()
net.clear_timer()
prog_bar.start((len(eval_dataset) + eval_input_cfg.batch_size - 1) // eval_input_cfg.batch_size)
for example in iter(eval_dataloader):
example = example_convert_to_torch(example, float_dtype)
if pickle_result:
results = predict_kitti_to_anno(
net, example, class_names, center_limit_range,
model_cfg.lidar_input)
dt_annos += results
else:
_predict_kitti_to_file(net, example, result_path_step,
class_names, center_limit_range,
model_cfg.lidar_input)
prog_bar.print_bar()
sec_per_ex = len(eval_dataset) / (time.time() - t)
print(f'generate label finished({sec_per_ex:.2f}/s). start eval:')
print(f'generate label finished({sec_per_ex:.2f}/s). start eval:',file=logf)
gt_annos = [
info["annos"] for info in eval_dataset.dataset.kitti_infos
]
if not pickle_result:
dt_annos = kitti.get_label_annos(result_path_step)
# result = get_official_eval_result_v2(gt_annos, dt_annos, class_names)
# print(json.dumps(result, indent=2), file=logf)
result = get_official_eval_result(gt_annos, dt_annos, class_names)
print(result, file=logf)
print(result)
result_1 = result.split("\n")[:5]
result_2 = result.split("\n")[10:15]
result_3 = result.split("\n")[20:25]
emh = ['0_easy', '1_mod', '2_hard']
result_save = result_1
for i in range(len(result_save)-1):
save_targ = result_save[i+1]
name_val = save_targ.split(':')[0].split(' ')[0]
value_val = save_targ.split(':')[1:]
for ev in range(3):
each_val = value_val[0].split(',')[ev]
merge_txt = 'AP_kitti/car_70/' + name_val+'/'+emh[ev]
try:
writer.add_scalar(merge_txt, float(each_val), global_step)
except:
abc=1
import pdb; pdb.set_trace()
abc=1
if pickle_result:
with open(result_path_step / "result.pkl", 'wb') as f:
pickle.dump(dt_annos, f)
writer.add_text('eval_result', result, global_step)
logger.info('Evaluation on validation set:')
# MOTA, MOTP, recall, prec, F1, fp, fn, id_switches = validate(
# val_dataset,
# net,
# str(0 + 1),
# config_tr,
# result_path_tr,
# part='val')
# print(MOTA, MOTP, recall, prec, F1, fp, fn, id_switches)
# curr_step = step
# if tb_logger is not None:
# tb_logger.add_scalar('prec', prec, curr_step)
# tb_logger.add_scalar('recall', recall, curr_step)
# tb_logger.add_scalar('mota', MOTA, curr_step)
# tb_logger.add_scalar('motp', MOTP, curr_step)
# tb_logger.add_scalar('fp', fp, curr_step)
# tb_logger.add_scalar('fn', fn, curr_step)
# tb_logger.add_scalar('f1', F1, curr_step)
# tb_logger.add_scalar('id_switches', id_switches, curr_step)
# if lr_scheduler is not None:
# tb_logger.add_scalar('lr', current_lr, curr_step)
# is_best = MOTA > best_mota
# best_mota = max(MOTA, best_mota)
# print(best_mota)
# import pdb; pdb.set_trace()
# save_checkpoint(
# { 'step': net.get_global_step(),
# 'score_arch': config_tr.model.score_arch,
# 'appear_arch': config_tr.model.appear_arch,
# 'best_mota': best_mota,
# 'state_dict': tracking_module.model.state_dict(),
# 'optimizer': tracking_module.optimizer.state_dict(),
# }, is_best, config_tr.save_path + '/ckpt')
# net.train()
# save model before exit
torchplus.train.save_models(model_dir, [net, optimizer],
net.get_global_step())
logf.close()
def train(config_path,
model_dir,
use_fusion=False,
use_ft=False,
use_second_stage=False,
use_endtoend=False,
result_path=None,
create_folder=False,
display_step=50,
summary_step=5,
local_rank=0,
pickle_result=True,
patchs=None):
"""train a VoxelNet model specified by a config file.
"""
if create_folder:
if pathlib.Path(model_dir).exists():
model_dir = torchplus.train.create_folder(model_dir)
patchs = patchs or []
model_dir = pathlib.Path(model_dir)
model_dir.mkdir(parents=True, exist_ok=True)
if result_path is None:
result_path = model_dir / 'results'
config_file_bkp = "pipeline.config"
config = pipeline_pb2.TrainEvalPipelineConfig()
with open(config_path, "r") as f:
proto_str = f.read()
text_format.Merge(proto_str, config)
for patch in patchs:
patch = "config." + patch
exec(patch)
shutil.copyfile(config_path, str(model_dir / config_file_bkp))
input_cfg = config.train_input_reader
eval_input_cfg = config.eval_input_reader
model_cfg = config.model.second
train_cfg = config.train_config
######################
# BUILD VOXEL GENERATOR
######################
voxel_generator = voxel_builder.build(model_cfg.voxel_generator)
######################
# BUILD TARGET ASSIGNER
######################
bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]]
box_coder = box_coder_builder.build(model_cfg.box_coder)
target_assigner_cfg = model_cfg.target_assigner
target_assigner = target_assigner_builder.build(target_assigner_cfg,
bv_range, box_coder)
class_names = target_assigner.classes
######################
# BUILD NET
######################
center_limit_range = model_cfg.post_center_limit_range
if use_second_stage:
net = second_2stage_builder.build(model_cfg, voxel_generator,
target_assigner)
if use_endtoend:
net = second_endtoend_builder.build(model_cfg, voxel_generator,
target_assigner)
else:
net = second_builder.build(model_cfg, voxel_generator, target_assigner)
net.cuda()
# import pdb; pdb.set_trace()
print("num_trainable parameters:", len(list(net.parameters())))
# for n, p in net.named_parameters():
# print(n, p.shape)
# pth_name = 'pre_weight/first_stage/fusion_split/voxelnet-35210.tckpt'
# # pth_name = 'pre_weight/first_stage/fusion_split/voxelnet-20130.tckpt'
# res_pre_weights = torch.load(pth_name)
# new_res_state_dict = OrderedDict()
# model_dict = net.state_dict()
# for k,v in res_pre_weights.items():
# if 'global_step' not in k:
# if 'dir' not in k:
# new_res_state_dict[k] = v
# model_dict.update(new_res_state_dict)
# net.load_state_dict(model_dict)
######################
if use_second_stage or use_endtoend:
if use_fusion:
# pth_name = 'pre_weight/8020/voxelnet-20130.tckpt'
pth_name = 'pre_weight/first_stage/fusion_split/voxelnet-35210.tckpt'
for i in range(30):
print(
'################## load Fusion First stage weight complete #######################'
)
else:
pth_name = 'pre_weight/first_stage/lidaronly/voxelnet-30950.tckpt'
for i in range(30):
print(
'################## load LiDAR Only First stage weight complete #######################'
)
res_pre_weights = torch.load(pth_name)
new_res_state_dict = OrderedDict()
model_dict = net.state_dict()
for k, v in res_pre_weights.items():
if 'global_step' not in k:
if 'dir' not in k:
new_res_state_dict[k] = v
model_dict.update(new_res_state_dict)
net.load_state_dict(model_dict)
############ load FPN18 pre-weight #############
if (use_fusion and not use_second_stage and not use_endtoend):
# if True:
# or (use_endtoend and use_fusion):
fpn_depth = 18
pth_name = 'pre_weight/FPN' + str(fpn_depth) + '_retinanet_968.pth'
res_pre_weights = torch.load(pth_name)
new_res_state_dict = OrderedDict()
model_dict = net.state_dict()
for k, v in res_pre_weights['state_dict'].items():
if ('regressionModel' not in k) and ('classificationModel'
not in k):
name = k.replace('module', 'rpn')
new_res_state_dict[name] = v
model_dict.update(new_res_state_dict)
net.load_state_dict(model_dict)
for i in range(30):
print('!!!!!!!!!!!!!!!!!! load FPN' + str(fpn_depth) +
' weight complete !!!!!!!!!!!!!!!!!!')
################################################
# BUILD OPTIMIZER
#####################
# we need global_step to create lr_scheduler, so restore net first.
torchplus.train.try_restore_latest_checkpoints(model_dir, [net])
gstep = net.get_global_step() - 1
optimizer_cfg = train_cfg.optimizer
if train_cfg.enable_mixed_precision:
net.half()
net.metrics_to_float()
net.convert_norm_to_float(net)
loss_scale = train_cfg.loss_scale_factor
mixed_optimizer = optimizer_builder.build(
optimizer_cfg,
net,
mixed=train_cfg.enable_mixed_precision,
loss_scale=loss_scale)
optimizer = mixed_optimizer
"""
if train_cfg.enable_mixed_precision:
mixed_optimizer = torchplus.train.MixedPrecisionWrapper(
optimizer, loss_scale)
else:
mixed_optimizer = optimizer
"""
# must restore optimizer AFTER using MixedPrecisionWrapper
torchplus.train.try_restore_latest_checkpoints(model_dir,
[mixed_optimizer])
lr_scheduler = lr_scheduler_builder.build(optimizer_cfg, optimizer,
train_cfg.steps)
if train_cfg.enable_mixed_precision:
float_dtype = torch.float16
else:
float_dtype = torch.float32
######################
# PREPARE INPUT
######################
dataset = input_reader_builder.build(input_cfg,
model_cfg,
training=True,
voxel_generator=voxel_generator,
target_assigner=target_assigner)
eval_dataset = input_reader_builder.build(eval_input_cfg,
model_cfg,
training=False,
voxel_generator=voxel_generator,
target_assigner=target_assigner)
def _worker_init_fn(worker_id):
time_seed = np.array(time.time(), dtype=np.int32)
np.random.seed(time_seed + worker_id)
print(f"WORKER {worker_id} seed:", np.random.get_state()[1][0])
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=input_cfg.batch_size,
shuffle=True,
num_workers=input_cfg.num_workers,
pin_memory=False,
collate_fn=merge_second_batch,
worker_init_fn=_worker_init_fn)
eval_dataloader = torch.utils.data.DataLoader(
eval_dataset,
batch_size=eval_input_cfg.batch_size,
shuffle=False,
num_workers=eval_input_cfg.num_workers,
pin_memory=False,
collate_fn=merge_second_batch)
data_iter = iter(dataloader)
######################
# TRAINING
######################
training_detail = []
log_path = model_dir / 'log.txt'
training_detail_path = model_dir / 'log.json'
if training_detail_path.exists():
with open(training_detail_path, 'r') as f:
training_detail = json.load(f)
logf = open(log_path, 'a')
logf.write(proto_str)
logf.write("\n")
summary_dir = model_dir / 'summary'
summary_dir.mkdir(parents=True, exist_ok=True)
writer = SummaryWriter(str(summary_dir))
total_step_elapsed = 0
remain_steps = train_cfg.steps - net.get_global_step()
t = time.time()
ckpt_start_time = t
total_loop = train_cfg.steps // train_cfg.steps_per_eval + 1
# total_loop = remain_steps // train_cfg.steps_per_eval + 1
clear_metrics_every_epoch = train_cfg.clear_metrics_every_epoch
if train_cfg.steps % train_cfg.steps_per_eval == 0:
total_loop -= 1
mixed_optimizer.zero_grad()
try:
for _ in range(total_loop):
if total_step_elapsed + train_cfg.steps_per_eval > train_cfg.steps:
steps = train_cfg.steps % train_cfg.steps_per_eval
else:
steps = train_cfg.steps_per_eval
for step in range(steps):
lr_scheduler.step(net.get_global_step())
try:
example = next(data_iter)
except StopIteration:
print("end epoch")
if clear_metrics_every_epoch:
net.clear_metrics()
data_iter = iter(dataloader)
example = next(data_iter)
example_torch = example_convert_to_torch(example, float_dtype)
batch_size = example["anchors"].shape[0]
ret_dict = net(example_torch)
# box_preds = ret_dict["box_preds"]
cls_preds = ret_dict["cls_preds"]
loss = ret_dict["loss"].mean()
cls_loss_reduced = ret_dict["cls_loss_reduced"].mean()
loc_loss_reduced = ret_dict["loc_loss_reduced"].mean()
cls_pos_loss = ret_dict["cls_pos_loss"]
cls_neg_loss = ret_dict["cls_neg_loss"]
loc_loss = ret_dict["loc_loss"]
cls_loss = ret_dict["cls_loss"]
dir_loss_reduced = ret_dict["dir_loss_reduced"]
cared = ret_dict["cared"]
# idx_offset = ret_dict["idx_offset"]
# labels = example_torch["labels"]
if use_second_stage or use_endtoend:
labels = ret_dict["labels"]
else:
labels = example_torch["labels"]
if train_cfg.enable_mixed_precision:
loss *= loss_scale
loss.backward()
# import pdb; pdb.set_trace()
torch.nn.utils.clip_grad_norm_(net.parameters(), 10.0)
mixed_optimizer.step()
mixed_optimizer.zero_grad()
net.update_global_step()
net_metrics = net.update_metrics(cls_loss_reduced,
loc_loss_reduced, cls_preds,
labels, cared)
step_time = (time.time() - t)
t = time.time()
metrics = {}
num_pos = int((labels > 0)[0].float().sum().cpu().numpy())
num_neg = int((labels == 0)[0].float().sum().cpu().numpy())
if 'anchors_mask' not in example_torch:
num_anchors = example_torch['anchors'].shape[1]
else:
num_anchors = int(example_torch['anchors_mask'][0].sum())
global_step = net.get_global_step()
# print(step)
if global_step % display_step == 0:
loc_loss_elem = [
float(loc_loss[:, :, i].sum().detach().cpu().numpy() /
batch_size) for i in range(loc_loss.shape[-1])
]
metrics["type"] = "step_info"
metrics["step"] = global_step
metrics["steptime"] = step_time
metrics.update(net_metrics)
metrics["loss"] = {}
metrics["loss"]["loc_elem"] = loc_loss_elem
metrics["loss"]["cls_pos_rt"] = float(
cls_pos_loss.detach().cpu().numpy())
metrics["loss"]["cls_neg_rt"] = float(
cls_neg_loss.detach().cpu().numpy())
if model_cfg.use_direction_classifier:
metrics["loss"]["dir_rt"] = float(
dir_loss_reduced.detach().cpu().numpy())
metrics["num_vox"] = int(example_torch["voxels"].shape[0])
metrics["num_pos"] = int(num_pos)
metrics["num_neg"] = int(num_neg)
metrics["num_anchors"] = int(num_anchors)
# metrics["idx_offset_mean"] = float(idx_offset.mean().detach().cpu().numpy())
# metrics["idx_offset_sum"] = float(idx_offset.sum().detach().cpu().numpy())
# metrics["lr"] = float(
# mixed_optimizer.param_groups[0]['lr'])
metrics["lr"] = float(optimizer.lr)
metrics["image_idx"] = example['image_idx'][0]
training_detail.append(metrics)
flatted_metrics = flat_nested_json_dict(metrics)
flatted_summarys = flat_nested_json_dict(metrics, "/")
for k, v in flatted_summarys.items():
if isinstance(v, (list, tuple)):
v = {str(i): e for i, e in enumerate(v)}
if type(v) != str and ('loc_elem' not in k):
writer.add_scalars(k, v, global_step)
else:
if (type(v) != str) and ('loc_elem' not in k):
writer.add_scalar(k, v, global_step)
# if use_second_stage or use_endtoend:
# bev_logs = ret_dict['bev_crops_output'][:64,0,...].view(64,1,14,14)
# bev_vis = torchvision.utils.make_grid(bev_logs,normalize=True,scale_each=True)
# writer.add_image('bev_crop',img_tensor=bev_vis, global_step=global_step)
# if ret_dict['concat_crops_output'] is not None:
# concat_logs = ret_dict['concat_crops_output'][:64,0,...].view(64,1,14,14)
# concat_vis = torchvision.utils.make_grid(concat_logs,normalize=True,scale_each=True)
# writer.add_image('concat_crop',img_tensor=concat_vis, global_step=global_step)
metrics_str_list = []
for k, v in flatted_metrics.items():
if isinstance(v, float):
metrics_str_list.append(f"{k}={v:.3}")
elif isinstance(v, (list, tuple)):
if v and isinstance(v[0], float):
v_str = ', '.join([f"{e:.3}" for e in v])
metrics_str_list.append(f"{k}=[{v_str}]")
else:
metrics_str_list.append(f"{k}={v}")
else:
metrics_str_list.append(f"{k}={v}")
log_str = ', '.join(metrics_str_list)
print(log_str, file=logf)
print(log_str)
ckpt_elasped_time = time.time() - ckpt_start_time
if ckpt_elasped_time > train_cfg.save_checkpoints_secs:
torchplus.train.save_models(model_dir, [net, optimizer],
net.get_global_step())
ckpt_start_time = time.time()
total_step_elapsed += steps
torchplus.train.save_models(model_dir, [net, optimizer],
net.get_global_step())
net.eval()
result_path_step = result_path / f"step_{net.get_global_step()}"
result_path_step.mkdir(parents=True, exist_ok=True)
print("#################################")
print("#################################", file=logf)
print("# EVAL")
print("# EVAL", file=logf)
print("#################################")
print("#################################", file=logf)
print("Generate output labels...")
print("Generate output labels...", file=logf)
t = time.time()
dt_annos = []
prog_bar = ProgressBar()
net.clear_timer()
prog_bar.start(
(len(eval_dataset) + eval_input_cfg.batch_size - 1) //
eval_input_cfg.batch_size)
for example in iter(eval_dataloader):
example = example_convert_to_torch(example, float_dtype)
if pickle_result:
dt_annos += predict_kitti_to_anno(net, example,
class_names,
center_limit_range,
model_cfg.lidar_input)
else:
_predict_kitti_to_file(net, example, result_path_step,
class_names, center_limit_range,
model_cfg.lidar_input)
prog_bar.print_bar()
sec_per_ex = len(eval_dataset) / (time.time() - t)
print(f'generate label finished({sec_per_ex:.2f}/s). start eval:')
print(f'generate label finished({sec_per_ex:.2f}/s). start eval:',
file=logf)
gt_annos = [
info["annos"] for info in eval_dataset.dataset.kitti_infos
]
if not pickle_result:
dt_annos = kitti.get_label_annos(result_path_step)
# result = get_official_eval_result_v2(gt_annos, dt_annos, class_names)
# print(json.dumps(result, indent=2), file=logf)
result = get_official_eval_result(gt_annos, dt_annos, class_names)
print(result, file=logf)
print(result)
result_1 = result.split("\n")[:5]
result_2 = result.split("\n")[10:15]
result_3 = result.split("\n")[20:25]
emh = ['0_easy', '1_mod', '2_hard']
result_save = result_1
for i in range(len(result_save) - 1):
save_targ = result_save[i + 1]
name_val = save_targ.split(':')[0].split(' ')[0]
value_val = save_targ.split(':')[1:]
for ev in range(3):
each_val = value_val[0].split(',')[ev]
merge_txt = 'AP_kitti/car_70/' + name_val + '/' + emh[ev]
writer.add_scalar(merge_txt, float(each_val), global_step)
if pickle_result:
with open(result_path_step / "result.pkl", 'wb') as f:
pickle.dump(dt_annos, f)
writer.add_text('eval_result', result, global_step)
net.train()
except Exception as e:
torchplus.train.save_models(model_dir, [net, optimizer],
net.get_global_step())
logf.close()
raise e
# save model before exit
torchplus.train.save_models(model_dir, [net, optimizer],
net.get_global_step())
logf.close()
def evaluation_from_kitti_dets(self, dt_annos, output_dir):
if "annos" not in self._kitti_infos[0]:
return None
gt_annos = [info["annos"] for info in self._kitti_infos]
# firstly convert standard detection to kitti-format dt annos
z_axis = 1 # KITTI camera format use y as regular "z" axis.
z_center = 1.0 # KITTI camera box's center is [0.5, 1, 0.5]
# for regular raw lidar data, z_axis = 2, z_center = 0.5.
result_official_dict = get_official_eval_result(
gt_annos,
dt_annos,
self._class_names,
z_axis=z_axis,
z_center=z_center)
result_coco = get_coco_eval_result(
gt_annos,
dt_annos,
self._class_names,
z_axis=z_axis,
z_center=z_center)
# feature extraction
for info, det in tqdm(zip(self._kitti_infos, dt_annos), desc="feature", total=len(dt_annos)):
pc_info = info["point_cloud"]
image_info = info["image"]
calib = info["calib"]
num_features = pc_info["num_features"]
v_path = self._root_path / pc_info["velodyne_path"]
v_path = str(v_path.parent.parent / (v_path.parent.stem + "_reduced") / v_path.name)
points_v = np.fromfile(
v_path, dtype=np.float32, count=-1).reshape([-1, num_features])
rect = calib['R0_rect']
Trv2c = calib['Tr_velo_to_cam']
P2 = calib['P2']
if False: # No longer you need remove outside image-rect (*_reduced pointcloud is already filtered.)
points_v = box_np_ops.remove_outside_points(
points_v, rect, Trv2c, P2, image_info["image_shape"])
annos = det
num_obj = len([n for n in annos['name'] if n != 'DontCare'])
# annos = kitti.filter_kitti_anno(annos, ['DontCare'])
dims = annos['dimensions'][:num_obj]
loc = annos['location'][:num_obj]
rots = annos['rotation_y'][:num_obj]
gt_boxes_camera = np.concatenate([loc, dims, rots[..., np.newaxis]],
axis=1)
gt_boxes_lidar = box_np_ops.box_camera_to_lidar(
gt_boxes_camera, rect, Trv2c)
indices = box_np_ops.points_in_rbbox(points_v[:, :3], gt_boxes_lidar)
num_points_in_gt = indices.sum(0)
num_ignored = len(annos['dimensions']) - num_obj
num_points_in_gt = np.concatenate(
[num_points_in_gt, -np.ones([num_ignored])])
annos["num_points_in_det"] = num_points_in_gt.astype(np.int32)
return {
"results": {
"official": result_official_dict["result"],
"coco": result_coco["result"],
},
"detail": {
"eval.kitti": {
"official": result_official_dict["detail"],
"coco": result_coco["detail"]
}
},
"result_kitti": result_official_dict["detections"],
}
def evaluate(config_path, model_id=None, from_file_mode = False, epoch_idx=None):
"""
Args:
config_path (str) : Path to the config yaml
model_id (int) : Which model id should be evaluated? "None" means the one in the config is used
from_file_mode (bool) : "True" means that eval data from a file is used
limit (int) : OPTIONS: None, int // after how many datapoints we want to exit
"""
print("**********************************************")
print("* Start Evaluation")
print("**********************************************")
# This variable "model_id_memory" just helps to remember the original state of "model_id"
model_id_memory = 0
if model_id == None: model_id_memory = None
# ------------------------------------------------------------------------------------------------------
# load the config from file and set Variables
# ------------------------------------------------------------------------------------------------------
with open(config_path) as f1:
config = yaml.load(f1, Loader=yaml.FullLoader)
# If no model_id is given we take the one from the config
if model_id == None: model_id = config["eval_model_id"]
eval_checkpoint = config["eval_checkpoint"]
print("**********************************************")
print("* Load Model ID {}".format(str(model_id)))
print("**********************************************")
# set training to false -> eval mode
training = False
# ------------------------------------------------------------------------------------------------------
# Load directory parameter and create directories
# ------------------------------------------------------------------------------------------------------
project_dir_base = config["project_dir_base"] # path to base project dir (where the whole code is stored)
# path to out base dir (where the training logs are stored)
out_dir_base = create_out_dir_base(project_dir_base, training, model_id)
# create the subdir where the training is stored
out_dir_eval_results, out_dir_checkpoints = create_model_dirs_eval(out_dir_base)
# ------------------------------------------------------------------------------------------------------
# Load dataloader parameter and create dataloader
# ------------------------------------------------------------------------------------------------------
limit = None # Options: {None,Int} # limit the amount of test data to be evaluated to save time
batch_size = config["eval_input_reader"]["batch_size"]
num_point_features = config["train_input_reader"]["num_point_features"]
center_limit_range = config["model"]["second"]["post_center_limit_range"]
desired_objects = config["eval_input_reader"]["desired_objects"]
no_annos_mode = config["eval_input_reader"]["no_annos_mode"]
production_mode = bool(config["production_mode"])
prediction_min_score = config["prediction_min_score"]
# create the dataLoader object which is reponsible for loading datapoints
# contains not much logic and basically just holds some variables
dataset_ori = dataLoader(training, None, config)
# initializes the dataset object (batch creating etc.)
dataset = dataset_ori.getIterator()
# makes the dataset object iterable
data_iterator = tf.compat.v1.data.make_one_shot_iterator(dataset)
# ------------------------------------------------------------------------------------------------------
# Create network
# ------------------------------------------------------------------------------------------------------
# create tensorboard writer for logging
writer = tf.summary.create_file_writer(out_dir_base)
# create network
net = VoxelNet(config, writer, training=training)
# DEBUG
if from_file_mode:
with open("eval_dataloader_limit200", "rb") as file:
data_iterator = pickle.load(file)
# ------------------------------------------------------------------------------------------------------
# We wrap a training step in a tf.function to to gain speedups (5x)
# ------------------------------------------------------------------------------------------------------
max_number_of_points_per_voxel = config["model"]["second"]["voxel_generator"]["max_number_of_points_per_voxel"]
@tf.function(input_signature = [tf.TensorSpec(shape=[None,max_number_of_points_per_voxel,num_point_features], dtype=tf.float32),tf.TensorSpec(shape=[None,], dtype=tf.int32),tf.TensorSpec(shape=[None,4], dtype=tf.int32),tf.TensorSpec(shape=[None,None,7], dtype=tf.float32)])
def trainStep(voxels,num_points,coors,batch_anchors):
preds_dict = net(voxels,num_points,coors,batch_anchors)
return preds_dict
# ------------------------------------------------------------------------------------------------------
# EPOCH ITERATOR Settings
# ------------------------------------------------------------------------------------------------------
# Helper Variable to load weights
load_weights_finished = False # This variable is part of a tensorflow workaround caused by subclassing keras.model
# # Create a model using low-level tf.* APIs
# class Squared(tf.Module):
# @tf.function
# def __call__(self, x):
# return tf.square(x)
# model = Squared()
# # (ro run your model) result = Squared(5.0) # This prints "25.0"
# # (to generate a SavedModel) tf.saved_model.save(model, "saved_model_tf_dir")
# concrete_func = model.__call__.get_concrete_function()
# # Convert the model
# converter = tf.lite.TFLiteConverter.from_concrete_functions([concrete_func])
# tflite_model = converter.convert()
# Helper Variable to save the output of the network
dt_annos = []
# eval params
measure_time = config["measure_time"]
# Helper Variables for Time Bechmarks
current_milli_time = lambda: int(round(time.time() * 1000))
t_full_sample_list = []
t_preprocess_list = []
t_network_list = []
t_predict_list = []
t_anno_list = []
t_rviz_list = []
# ------------------------------------------------------------------------------------------------------
# ROS
# ------------------------------------------------------------------------------------------------------
if production_mode:
bb_pred_guess_1_pub = rospy.Publisher("bb_pred_guess_1", BoundingBoxArray)
header = std_msgs.msg.Header()
header.stamp = rospy.Time.now()
header.frame_id = 'camera_color_frame'
calib = {"R0_rect" : np.array([1.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 1.0]).reshape(3,3),
"Tr_velo_to_cam" : np.array([0.0, -1.0, 0.0, 0.0, 0.0, 0.0, -1.0, 0.0, 1.0, 0.0, 0.0, 0.0]).reshape(3,4)}
# ------------------------------------------------------------------------------------------------------
# EPOCH ITERATOR
# ------------------------------------------------------------------------------------------------------
for i, example in enumerate(data_iterator):
# example: [0:voxels, 1:num_points, 2:coordinates, 3:rect, 4:Trv2c, 5:P2, 6:anchors, 7:anchors_mask, 8:image_idx, 9:image_shape]
if measure_time:
if i > 0:
t_preprocess = current_milli_time() - t_preprocess
t_preprocess_list.append(t_preprocess)
else:
t_preprocess_list.append(0.0)
# save starting time to measure network speed
if measure_time: t_full_sample = current_milli_time()
# Progess Bar
if not production_mode:
if limit is None:
progressBar(i,dataset_ori.ndata//batch_size)
else:
progressBar(i,limit)
if i == limit: break # consider the case for early exit
# ------------------------------------------------------------------------------------------------------
# Load Model Weights and Optimizer Weights (Checkpoint)
# ------------------------------------------------------------------------------------------------------
if load_weights_finished == False: # support variable, to check if the weights are loaded
# initialize the model # TODO delete?
#net(example[0],example[1],example[2],example[6],example[8],example[9])
net(example[0],example[1],example[2],example[6])
load_weights_finished = True
# load the weights depending on if we are in training mode since
# if yes the "model_weights_temp" file needs to be evaluated
model_dir = ""
if model_id_memory == None: # for explaination, see variable "model_id_memory"
model_dir = out_dir_checkpoints + eval_checkpoint
net.load_weights(model_dir)
else:
model_dir = out_dir_checkpoints + "/model_weights_temp.h5"
net.load_weights(model_dir)
print("**********************************************")
print("* Model Loaded from Path: {}".format(model_dir))
print("**********************************************")
# ------------------------------------------------------------------------------------------------------
# Run Network
# ------------------------------------------------------------------------------------------------------
if measure_time: t_network = current_milli_time()
preds_dict = trainStep(example[0],example[1],example[2],example[6])
if measure_time:
t_network = current_milli_time() - t_network
t_network_list.append(t_network)
# ------------------------------------------------------------------------------------------------------
# Convert Network Output to predictions by applying the direction classifier to predictions of rotation
# use nms to get the final bboxes
# ------------------------------------------------------------------------------------------------------
if measure_time: t_predict = current_milli_time()
# t_full_sample: 23.91, t_preprocess: 0.85, t_network: 12.54, t_predict: 10.46, t_anno: 0.84, t_rviz: 0.0
# t_full_sample: 23.54, t_preprocess: 1.13, t_network: 11.08, t_predict: 11.45, t_anno: 0.82, t_rviz: 0.01
predictions_dicts = net.predict(example,preds_dict) # list(predictions) of ['name', 'truncated', 'occluded', 'alpha', 'bbox', 'dimensions', 'location', 'rotation_y', 'score', 'image_idx'])
if measure_time:
t_predict = current_milli_time() -t_predict
t_predict_list.append(t_predict)
# ------------------------------------------------------------------------------------------------------
# Convert Predictions to Kitti Annotation style
# ------------------------------------------------------------------------------------------------------
if measure_time: t_anno = current_milli_time()
dt_anno = predict_kitti_to_anno(example, desired_objects, predictions_dicts, None, False)
if measure_time:
t_anno = current_milli_time() -t_anno
t_anno_list.append(t_anno)
# ------------------------------------------------------------------------------------------------------
# Send annotation to RVIZ
# ------------------------------------------------------------------------------------------------------
if measure_time: t_rviz = current_milli_time()
if production_mode:
dt_anno = remove_low_score(dt_anno[0], float(prediction_min_score))
# if len(dt_anno["score"]) > 0:
# print(dt_anno["score"])
dims = dt_anno['dimensions']
loc = dt_anno['location']
rots = dt_anno['rotation_y']
boxes_camera = np.concatenate([loc, dims, rots[..., np.newaxis]], axis=1)
boxes_lidar = box_camera_to_lidar(boxes_camera, calib['R0_rect'],calib['Tr_velo_to_cam'])
centers,dims,angles = boxes_lidar[:, :3], boxes_lidar[:, 3:6], boxes_lidar[:, 6] # [a,b,c] -> [c,a,b] (camera to lidar coords)
# LIFT bounding boxes
# - the postition of bboxes in the pipeline is at the z buttom of the bb and ros needs it at the z center
# - TODO: lift by height/2 and not just 1.0
#centers = centers + [0.0,0.0,0.9]
send_3d_bbox(centers, dims, angles, bb_pred_guess_1_pub, header)
else:
dt_annos += dt_anno
if measure_time:
t_rviz = current_milli_time() -t_rviz
t_rviz_list.append(t_rviz)
t_full_sample = current_milli_time() -t_full_sample
t_full_sample_list.append(t_full_sample)
t_preprocess = current_milli_time()
# ------------------------------------------------------------------------------------------------------
# Print times
# ------------------------------------------------------------------------------------------------------
if i > 0 and measure_time: # we scipt the first network iteration (initialization)
t_full_sample_avg = round(sum(t_full_sample_list[1:])/len(t_full_sample_list[1:]),2)
t_preprocess_avg = round(sum(t_preprocess_list[1:])/len(t_preprocess_list[1:]),2)
t_network_avg = round(sum(t_network_list[1:])/len(t_network_list[1:]),2)
t_predict_avg = round(sum(t_predict_list[1:])/len(t_predict_list[1:]),2)
t_anno_avg = round(sum(t_anno_list[1:])/len(t_anno_list[1:]),2)
t_rviz_avg = round(sum(t_rviz_list[1:])/len(t_rviz_list[1:]),2)
print(f't_full_sample: {t_full_sample_avg}, t_preprocess: {t_preprocess_avg}, t_network: {t_network_avg}, t_predict: {t_predict_avg}, t_anno: {t_anno_avg}, t_rviz: {t_rviz_avg}')
# ------------------------------------------------------------------------------------------------------
# save the results in a file
# ------------------------------------------------------------------------------------------------------
if epoch_idx is not None: # if epoch index is given include in the name (typically evaluation during training)
with open(out_dir_eval_results + "/result_epoch_{}.pkl".format(str(epoch_idx)), 'wb') as f:
pickle.dump(dt_annos, f, 2)
else: # if epoch index is not given use generic name (typically while evaluation of certain models, epochs, and testing sets)
with open(out_dir_eval_results + "/result.pkl", 'wb') as f:
pickle.dump(dt_annos, f, 2)
# ------------------------------------------------------------------------------------------------------
# Exit the program if we run in no_annos_mode (since we dont have annotations we cannot do the following evaluation.
# ------------------------------------------------------------------------------------------------------
if no_annos_mode:
return (np.array([0]),"no evaluation")
# get all gt in dataset
gt_annos = [info["annos"] for info in dataset_ori.img_list_and_infos]
# DEBUG
# This can limit the amount of test data to be evaluated to save time
if limit is not None:
gt_annos = gt_annos[0:limit]
else:
gt_annos = gt_annos[0:len(dt_annos)]
# ------------------------------------------------------------------------------------------------------
# evaluate the predictions in KITTI? style
# - AOS is average orientation similarity, bbox is 2D
# - Results for the columns (difficulties) will be equal since we do not have OCCLUSION and TRUNCATION
# annos in out ground truth which have influence on the difficulties
# - Input: dt_annos, gt_annos are in camera coors (in Lidar expression: (-y,-z,x))
# ------------------------------------------------------------------------------------------------------
compute_bbox = False # Since we dont have 2D box ground truth we don want to compute 2D bboxes results
result1, mAPbbox, mAPbev, mAP3d, mAPaos = get_official_eval_result(gt_annos, dt_annos, desired_objects, compute_bbox = compute_bbox)
# ------------------------------------------------------------------------------------------------------
# Print the evaluation result which depicts the overlaps between the gt and predictions
# - if multiple evals of the same class (e.g. Pedestrian,Pedestrian) occur, these depict separate evaluations
# with different >overlap settings<
# - in each of those evaluations: columns are difficulties (dependent on OCCLUSION and TRUNCATION) and
# rows are [bbox,bev,3D]. The >overlap settings< are specified right to the class name e.g.:
# Pedestrian [email protected], 0.25, 0.25 -> [bbox, bev, 3D] and refer to the rows (NOT colums!). In this example
# bbox is 0.50, bev is 0.25 and 3D is 0.25 overlap. Also note, that bbox is missing in our evaluation.
# ------------------------------------------------------------------------------------------------------
print(result1)
# ------------------------------------------------------------------------------------------------------
# evaluate the predictions in COCO? style
# ------------------------------------------------------------------------------------------------------
# result2 = get_coco_eval_result(gt_annos, dt_annos, desired_objects)
# print(result2)
# ------------------------------------------------------------------------------------------------------
# return results (only used during training)
# ------------------------------------------------------------------------------------------------------
return (mAP3d,result1)
def evaluate(config_path,
model_dir,
result_path=None,
predict_test=False,
ckpt_path=None,
ref_detfile=None,
pickle_result=True,
measure_time=False,
batch_size=None):
model_dir = pathlib.Path(model_dir)
print("Predict_test: ", predict_test)
if predict_test:
result_name = 'predict_test'
else:
result_name = 'eval_results'
if result_path is None:
result_path = model_dir / result_name
else:
result_path = pathlib.Path(result_path)
config = pipeline_pb2.TrainEvalPipelineConfig()
with open(config_path, "r") as f:
proto_str = f.read()
text_format.Merge(proto_str, config)
input_cfg = config.eval_input_reader
model_cfg = config.model.second
train_cfg = config.train_config
detection_2d_path = config.train_config.detection_2d_path
center_limit_range = model_cfg.post_center_limit_range
######################
# BUILD VOXEL GENERATOR
######################
voxel_generator = voxel_builder.build(model_cfg.voxel_generator)
bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]]
box_coder = box_coder_builder.build(model_cfg.box_coder)
target_assigner_cfg = model_cfg.target_assigner
target_assigner = target_assigner_builder.build(target_assigner_cfg,
bv_range, box_coder)
class_names = target_assigner.classes
# this one is used for training car detector
net = build_inference_net('./configs/car.fhd.config', '../model_dir')
fusion_layer = fusion.fusion()
fusion_layer.cuda()
net.cuda()
############ restore parameters for fusion layer
if ckpt_path is None:
print("load existing model for fusion layer")
torchplus.train.try_restore_latest_checkpoints(model_dir,
[fusion_layer])
else:
torchplus.train.restore(ckpt_path, fusion_layer)
if train_cfg.enable_mixed_precision:
net.half()
net.metrics_to_float()
net.convert_norm_to_float(net)
batch_size = batch_size or input_cfg.batch_size
eval_dataset = input_reader_builder.build(input_cfg,
model_cfg,
training=not predict_test,
voxel_generator=voxel_generator,
target_assigner=target_assigner)
eval_dataloader = torch.utils.data.DataLoader(
eval_dataset,
batch_size=batch_size,
shuffle=False,
num_workers=0, # input_cfg.num_workers,
pin_memory=False,
collate_fn=merge_second_batch)
if train_cfg.enable_mixed_precision:
float_dtype = torch.float16
else:
float_dtype = torch.float32
net.eval()
fusion_layer.eval()
result_path_step = result_path / f"step_{net.get_global_step()}"
result_path_step.mkdir(parents=True, exist_ok=True)
t = time.time()
dt_annos = []
global_set = None
print("Generate output labels...")
bar = ProgressBar()
bar.start((len(eval_dataset) + batch_size - 1) // batch_size)
prep_example_times = []
prep_times = []
t2 = time.time()
val_loss_final = 0
for example in iter(eval_dataloader):
if measure_time:
prep_times.append(time.time() - t2)
t1 = time.time()
torch.cuda.synchronize()
example = example_convert_to_torch(example, float_dtype)
if measure_time:
torch.cuda.synchronize()
prep_example_times.append(time.time() - t1)
if pickle_result:
dt_annos_i, val_losses = predict_kitti_to_anno(
net, detection_2d_path, fusion_layer, example, class_names,
center_limit_range, model_cfg.lidar_input, global_set)
dt_annos += dt_annos_i
val_loss_final = val_loss_final + val_losses
else:
_predict_kitti_to_file(net, detection_2d_path, fusion_layer,
example, result_path_step, class_names,
center_limit_range, model_cfg.lidar_input)
bar.print_bar()
if measure_time:
t2 = time.time()
sec_per_example = len(eval_dataset) / (time.time() - t)
print(f'generate label finished({sec_per_example:.2f}/s). start eval:')
print("validation_loss:", val_loss_final / len(eval_dataloader))
if measure_time:
print(
f"avg example to torch time: {np.mean(prep_example_times) * 1000:.3f} ms"
)
print(f"avg prep time: {np.mean(prep_times) * 1000:.3f} ms")
for name, val in net.get_avg_time_dict().items():
print(f"avg {name} time = {val * 1000:.3f} ms")
if not predict_test:
gt_annos = [info["annos"] for info in eval_dataset.dataset.kitti_infos]
if not pickle_result:
dt_annos = kitti.get_label_annos(result_path_step)
result = get_official_eval_result(gt_annos, dt_annos, class_names)
# print(json.dumps(result, indent=2))
print(result)
result = get_coco_eval_result(gt_annos, dt_annos, class_names)
print(result)
if pickle_result:
with open(result_path_step / "result.pkl", 'wb') as f:
pickle.dump(dt_annos, f)
else:
if pickle_result:
with open(result_path_step / "result.pkl", 'wb') as f:
pickle.dump(dt_annos, f)
def train(config_path,
model_dir,
result_path=None,
create_folder=False,
display_step=50,
summary_step=5,
pickle_result=True,
patchs=None):
torch.manual_seed(3)
np.random.seed(3)
if create_folder:
if pathlib.Path(model_dir).exists():
model_dir = torchplus.train.create_folder(model_dir)
patchs = patchs or []
model_dir = pathlib.Path(model_dir)
model_dir.mkdir(parents=True, exist_ok=True)
if result_path is None:
result_path = model_dir / 'results'
config = pipeline_pb2.TrainEvalPipelineConfig()
with open(config_path, "r") as f:
proto_str = f.read()
text_format.Merge(proto_str, config)
input_cfg = config.train_input_reader
eval_input_cfg = config.eval_input_reader
model_cfg = config.model.second
train_cfg = config.train_config
detection_2d_path = config.train_config.detection_2d_path
print("2d detection path:", detection_2d_path)
center_limit_range = model_cfg.post_center_limit_range
voxel_generator = voxel_builder.build(model_cfg.voxel_generator)
bv_range = voxel_generator.point_cloud_range[[0, 1, 3, 4]]
box_coder = box_coder_builder.build(model_cfg.box_coder)
target_assigner_cfg = model_cfg.target_assigner
target_assigner = target_assigner_builder.build(target_assigner_cfg,
bv_range, box_coder)
class_names = target_assigner.classes
net = build_inference_net('./configs/car.fhd.config', '../model_dir')
fusion_layer = fusion.fusion()
fusion_layer.cuda()
optimizer_cfg = train_cfg.optimizer
if train_cfg.enable_mixed_precision:
net.half()
net.metrics_to_float()
net.convert_norm_to_float(net)
loss_scale = train_cfg.loss_scale_factor
mixed_optimizer = optimizer_builder.build(
optimizer_cfg,
fusion_layer,
mixed=train_cfg.enable_mixed_precision,
loss_scale=loss_scale)
optimizer = mixed_optimizer
# must restore optimizer AFTER using MixedPrecisionWrapper
torchplus.train.try_restore_latest_checkpoints(model_dir,
[mixed_optimizer])
lr_scheduler = lr_scheduler_builder.build(optimizer_cfg, optimizer,
train_cfg.steps)
if train_cfg.enable_mixed_precision:
float_dtype = torch.float16
else:
float_dtype = torch.float32
######################
# PREPARE INPUT
######################
dataset = input_reader_builder.build(input_cfg,
model_cfg,
training=True,
voxel_generator=voxel_generator,
target_assigner=target_assigner)
eval_dataset = input_reader_builder.build(
eval_input_cfg,
model_cfg,
training=True, #if rhnning for test, here it needs to be False
voxel_generator=voxel_generator,
target_assigner=target_assigner)
def _worker_init_fn(worker_id):
time_seed = np.array(time.time(), dtype=np.int32)
np.random.seed(time_seed + worker_id)
print(f"WORKER {worker_id} seed:", np.random.get_state()[1][0])
dataloader = torch.utils.data.DataLoader(dataset,
batch_size=input_cfg.batch_size,
shuffle=True,
num_workers=input_cfg.num_workers,
pin_memory=False,
collate_fn=merge_second_batch,
worker_init_fn=_worker_init_fn)
eval_dataloader = torch.utils.data.DataLoader(
eval_dataset,
batch_size=eval_input_cfg.batch_size,
shuffle=False,
num_workers=eval_input_cfg.num_workers,
pin_memory=False,
collate_fn=merge_second_batch)
data_iter = iter(dataloader)
######################
# TRAINING
######################
focal_loss = SigmoidFocalClassificationLoss()
cls_loss_sum = 0
training_detail = []
log_path = model_dir / 'log.txt'
training_detail_path = model_dir / 'log.json'
if training_detail_path.exists():
with open(training_detail_path, 'r') as f:
training_detail = json.load(f)
logf = open(log_path, 'a')
logf.write(proto_str)
logf.write("\n")
summary_dir = model_dir / 'summary'
summary_dir.mkdir(parents=True, exist_ok=True)
writer = SummaryWriter(str(summary_dir))
total_step_elapsed = 0
remain_steps = train_cfg.steps - net.get_global_step()
t = time.time()
ckpt_start_time = t
total_loop = train_cfg.steps // train_cfg.steps_per_eval + 1
#print("steps, steps_per_eval, total_loop:", train_cfg.steps, train_cfg.steps_per_eval, total_loop)
# total_loop = remain_steps // train_cfg.steps_per_eval + 1
clear_metrics_every_epoch = train_cfg.clear_metrics_every_epoch
net.set_global_step(torch.tensor([0]))
if train_cfg.steps % train_cfg.steps_per_eval == 0:
total_loop -= 1
mixed_optimizer.zero_grad()
try:
for _ in range(total_loop):
if total_step_elapsed + train_cfg.steps_per_eval > train_cfg.steps:
steps = train_cfg.steps % train_cfg.steps_per_eval
else:
steps = train_cfg.steps_per_eval
for step in range(steps):
lr_scheduler.step(net.get_global_step())
try:
example = next(data_iter)
except StopIteration:
print("end epoch")
if clear_metrics_every_epoch:
net.clear_metrics()
data_iter = iter(dataloader)
example = next(data_iter)
example_torch = example_convert_to_torch(example, float_dtype)
batch_size = example["anchors"].shape[0]
all_3d_output_camera_dict, all_3d_output, top_predictions, fusion_input, tensor_index = net(
example_torch, detection_2d_path)
d3_gt_boxes = example_torch["d3_gt_boxes"][0, :, :]
if d3_gt_boxes.shape[0] == 0:
target_for_fusion = np.zeros((1, 70400, 1))
positives = torch.zeros(1,
70400).type(torch.float32).cuda()
negatives = torch.zeros(1,
70400).type(torch.float32).cuda()
negatives[:, :] = 1
else:
d3_gt_boxes_camera = box_torch_ops.box_lidar_to_camera(
d3_gt_boxes, example_torch['rect'][0, :],
example_torch['Trv2c'][0, :])
d3_gt_boxes_camera_bev = d3_gt_boxes_camera[:, [
0, 2, 3, 5, 6
]]
###### predicted bev boxes
pred_3d_box = all_3d_output_camera_dict[0]["box3d_camera"]
pred_bev_box = pred_3d_box[:, [0, 2, 3, 5, 6]]
#iou_bev = bev_box_overlap(d3_gt_boxes_camera_bev.detach().cpu().numpy(), pred_bev_box.detach().cpu().numpy(), criterion=-1)
iou_bev = d3_box_overlap(
d3_gt_boxes_camera.detach().cpu().numpy(),
pred_3d_box.squeeze().detach().cpu().numpy(),
criterion=-1)
iou_bev_max = np.amax(iou_bev, axis=0)
#print(np.max(iou_bev_max))
target_for_fusion = ((iou_bev_max >= 0.7) * 1).reshape(
1, -1, 1)
positive_index = ((iou_bev_max >= 0.7) * 1).reshape(1, -1)
positives = torch.from_numpy(positive_index).type(
torch.float32).cuda()
negative_index = ((iou_bev_max <= 0.5) * 1).reshape(1, -1)
negatives = torch.from_numpy(negative_index).type(
torch.float32).cuda()
cls_preds, flag = fusion_layer(fusion_input.cuda(),
tensor_index.cuda())
one_hot_targets = torch.from_numpy(target_for_fusion).type(
torch.float32).cuda()
negative_cls_weights = negatives.type(torch.float32) * 1.0
cls_weights = negative_cls_weights + 1.0 * positives.type(
torch.float32)
pos_normalizer = positives.sum(1, keepdim=True).type(
torch.float32)
cls_weights /= torch.clamp(pos_normalizer, min=1.0)
if flag == 1:
cls_losses = focal_loss._compute_loss(
cls_preds, one_hot_targets,
cls_weights.cuda()) # [N, M]
cls_losses_reduced = cls_losses.sum(
) / example_torch['labels'].shape[0]
cls_loss_sum = cls_loss_sum + cls_losses_reduced
if train_cfg.enable_mixed_precision:
loss *= loss_scale
cls_losses_reduced.backward()
mixed_optimizer.step()
mixed_optimizer.zero_grad()
net.update_global_step()
step_time = (time.time() - t)
t = time.time()
metrics = {}
global_step = net.get_global_step()
if global_step % display_step == 0:
print("now it is",
global_step,
"steps",
" and the cls_loss is :",
cls_loss_sum / display_step,
"learning_rate: ",
float(optimizer.lr),
file=logf)
print("now it is", global_step, "steps",
" and the cls_loss is :",
cls_loss_sum / display_step, "learning_rate: ",
float(optimizer.lr))
cls_loss_sum = 0
ckpt_elasped_time = time.time() - ckpt_start_time
if ckpt_elasped_time > train_cfg.save_checkpoints_secs:
torchplus.train.save_models(model_dir,
[fusion_layer, optimizer],
net.get_global_step())
ckpt_start_time = time.time()
total_step_elapsed += steps
torchplus.train.save_models(model_dir, [fusion_layer, optimizer],
net.get_global_step())
fusion_layer.eval()
net.eval()
result_path_step = result_path / f"step_{net.get_global_step()}"
result_path_step.mkdir(parents=True, exist_ok=True)
print("#################################")
print("#################################", file=logf)
print("# EVAL")
print("# EVAL", file=logf)
print("#################################")
print("#################################", file=logf)
print("Generate output labels...")
print("Generate output labels...", file=logf)
t = time.time()
dt_annos = []
prog_bar = ProgressBar()
net.clear_timer()
prog_bar.start(
(len(eval_dataset) + eval_input_cfg.batch_size - 1) //
eval_input_cfg.batch_size)
val_loss_final = 0
for example in iter(eval_dataloader):
example = example_convert_to_torch(example, float_dtype)
if pickle_result:
dt_annos_i, val_losses = predict_kitti_to_anno(
net, detection_2d_path, fusion_layer, example,
class_names, center_limit_range, model_cfg.lidar_input)
dt_annos += dt_annos_i
val_loss_final = val_loss_final + val_losses
else:
_predict_kitti_to_file(net, detection_2d_path, example,
result_path_step, class_names,
center_limit_range,
model_cfg.lidar_input)
prog_bar.print_bar()
sec_per_ex = len(eval_dataset) / (time.time() - t)
print("validation_loss:", val_loss_final / len(eval_dataloader))
print("validation_loss:",
val_loss_final / len(eval_dataloader),
file=logf)
print(f'generate label finished({sec_per_ex:.2f}/s). start eval:')
print(f'generate label finished({sec_per_ex:.2f}/s). start eval:',
file=logf)
gt_annos = [
info["annos"] for info in eval_dataset.dataset.kitti_infos
]
if not pickle_result:
dt_annos = kitti.get_label_annos(result_path_step)
# result = get_official_eval_result_v2(gt_annos, dt_annos, class_names)
result = get_official_eval_result(gt_annos, dt_annos, class_names)
print(result, file=logf)
print(result)
writer.add_text('eval_result', json.dumps(result, indent=2),
global_step)
result = get_coco_eval_result(gt_annos, dt_annos, class_names)
print(result, file=logf)
print(result)
if pickle_result:
with open(result_path_step / "result.pkl", 'wb') as f:
pickle.dump(dt_annos, f)
writer.add_text('eval_result', result, global_step)
#net.train()
fusion_layer.train()
except Exception as e:
torchplus.train.save_models(model_dir, [fusion_layer, optimizer],
net.get_global_step())
logf.close()
raise e
# save model before exit
torchplus.train.save_models(model_dir, [fusion_layer, optimizer],
net.get_global_step())
logf.close()
