def test_compute_ewc_weights_v1(self):
return
from incdet3.models.ewc_func import (
_init_ewc_weights, _cycle_next, _update_ewc_term,
_compute_accum_grad_v1, _compute_FIM_cls2term_v1,
_compute_FIM_reg2term_v1, _compute_FIM_clsregterm_v1,
_update_ewc_weights_v1)
from incdet3.builders.dataloader_builder import example_convert_to_torch
from tqdm import tqdm
from det3.ops import write_pkl
state = np.random.get_state()
torch_state_cpu = torch.Generator().get_state()
torch_state_gpu = torch.Generator(device="cuda:0").get_state()
network = build_network().cuda()
dataloader = build_dataloader()
num_of_datasamples = len(dataloader)
debug_mode = False
reg2_coef = 0.1
clsreg_coef = 0.1
est_ewc_weights = network.compute_ewc_weights_v1(
dataloader,
num_of_datasamples,
reg2_coef=reg2_coef,
clsreg_coef=clsreg_coef,
debug_mode=debug_mode)
# compute gt
gt_ewc_weights = _init_ewc_weights(network._model)
network.eval()
for i, data in tqdm(enumerate(dataloader)):
data = example_convert_to_torch(data,
dtype=torch.float32,
device=torch.device("cuda:0"))
voxels = data["voxels"]
num_points = data["num_points"]
coors = data["coordinates"]
batch_anchors = data["anchors"]
preds_dict = network._network_forward(network._model, voxels,
num_points, coors, 1)
box_preds = preds_dict["box_preds"]
cls_preds = preds_dict["cls_preds"]
labels = data['labels']
reg_targets = data['reg_targets']
importance = data['importance']
weights = Network._prepare_loss_weights(
labels,
pos_cls_weight=network._pos_cls_weight,
neg_cls_weight=network._neg_cls_weight,
loss_norm_type=network._loss_norm_type,
importance=importance,
use_direction_classifier=True,
dtype=box_preds.dtype)
cls_targets = labels * weights["cared"].type_as(labels)
cls_targets = cls_targets.unsqueeze(-1)
loss_cls = network._compute_classification_loss(
est=cls_preds,
gt=cls_targets,
weights=weights["cls_weights"] *
importance) * network._cls_loss_weight
loss_reg = network._compute_location_loss(
est=box_preds,
gt=reg_targets,
weights=weights["reg_weights"] *
importance) * network._loc_loss_weight
accum_grad_dict = _compute_accum_grad_v1(loss_cls, loss_reg,
network._model)
cls2_term = _compute_FIM_cls2term_v1(accum_grad_dict["cls_grad"])
reg2_term = _compute_FIM_reg2term_v1(accum_grad_dict["reg_grad"])
clsreg_term = _compute_FIM_clsregterm_v1(
accum_grad_dict["cls_grad"], accum_grad_dict["reg_grad"])
gt_ewc_weights = _update_ewc_weights_v1(gt_ewc_weights, cls2_term,
reg2_term, clsreg_term,
reg2_coef, clsreg_coef, i)
for name, param in gt_ewc_weights.items():
self.assertTrue(torch.allclose(param, est_ewc_weights[name]))
np.random.set_state(state)
torch.Generator().set_state(torch_state_cpu)
torch.Generator(device="cuda:0").set_state(torch_state_gpu)
def build_network():
network_cfg_template = {
"VoxelEncoder": {
"name": "SimpleVoxel",
"@num_input_features": 4,
},
"MiddleLayer": {
"name": "SpMiddleFHD",
"@use_norm": True,
"@num_input_features": 4,
"@output_shape": [1, 41, 1600, 1408, 16], #TBD
"downsample_factor": 8
},
"RPN": {
"name": "ResNetRPN",
"@use_norm": True,
"@num_class": None, # TBD
"@layer_nums": [5],
"@layer_strides": [1],
"@num_filters": [128],
"@upsample_strides": [1],
"@num_upsample_filters": [128],
"@num_input_features": 128,
"@num_anchor_per_loc": None, # TBD
"@encode_background_as_zeros": True,
"@use_direction_classifier": True,
"@use_groupnorm": False,
"@num_groups": 0,
"@box_code_size": 7, # TBD
"@num_direction_bins": 2,
},
}
name_template = "IncDetTest"
rpn_name = "ResNetRPN"
network_cfg = deepcopy(network_cfg_template)
network_cfg["RPN"]["name"] = rpn_name
network_cfg["RPN"]["@num_class"] = 3
network_cfg["RPN"]["@num_anchor_per_loc"] = 6
params = {
"classes_target": ["class1", "class2", "class3"],
"classes_source": ["class1", "class2"],
"model_resume_dict": {
"ckpt_path":
"unit_tests/data/train_class2-23200.tckpt",
"num_classes":
2,
"num_anchor_per_loc":
4,
"partially_load_params": [
"rpn.conv_cls.weight",
"rpn.conv_cls.bias",
"rpn.conv_box.weight",
"rpn.conv_box.bias",
"rpn.conv_dir_cls.weight",
"rpn.conv_dir_cls.bias",
]
},
"sub_model_resume_dict": {
"ckpt_path": "unit_tests/data/train_class2-23200.tckpt",
"num_classes": 2,
"num_anchor_per_loc": 4,
"partially_load_params": []
},
"voxel_encoder_dict": network_cfg["VoxelEncoder"],
"middle_layer_dict": network_cfg["MiddleLayer"],
"rpn_dict": network_cfg["RPN"],
"training_mode": "lwf",
"is_training": True,
"pos_cls_weight": 1.0,
"neg_cls_weight": 1.0,
"l2sp_alpha_coef": 2.0,
"weight_decay_coef": 0.01,
"delta_coef": 4.0,
"ewc_coef": 2.0,
"ewc_weights_path":
"unit_tests/data/test_model-ewc-ewc_weights_v1.pkl",
"distillation_loss_cls_coef": 1.0,
"distillation_loss_reg_coef": 1.0,
"num_biased_select": 2,
"loss_dict": {
"ClassificationLoss": {
"name": "SigmoidFocalClassificationLoss",
"@alpha": 0.25,
"@gamma": 2.0,
},
"LocalizationLoss": {
"name": "WeightedSmoothL1LocalizationLoss",
"@sigma": 3.0,
"@code_weights": [2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0],
"@codewise": True,
},
"DirectionLoss": {
"name": "WeightedSoftmaxClassificationLoss",
},
"DistillationClassificationLoss": {
"name": "WeightedSmoothL1LocalizationLoss",
"@sigma": 1.3,
"@code_weights": [4.0] * 2,
"@codewise": True,
},
"DistillationRegressionLoss": {
"name": "WeightedSmoothL1LocalizationLoss",
"@sigma": 3.0,
"@code_weights": [3.0] * 7,
"@codewise": True,
},
},
"hook_layers": [],
"distillation_mode": ["ewc"],
"bool_reuse_anchor_for_cls": False,
"bool_biased_select_with_submodel": False
}
network = Network(**params).cuda()
return network
def train(cfg):
global g_log_dir, g_save_dir
cores = setup_cores(cfg, mode="train")
model = cores["model"]
dataloader_train = cores["dataloader_train"]
dataloader_val = cores["dataloader_val"]
optimizer = cores["optimizer"]
lr_scheduler = cores["lr_scheduler"]
max_iter = cfg.TRAIN["train_iter"]
num_log_iter = cfg.TRAIN["num_log_iter"]
num_val_iter = cfg.TRAIN["num_val_iter"]
num_save_iter = cfg.TRAIN["num_save_iter"]
dataitr_train = dataloader_train.__iter__()
iter_elapsed = 0
while model.get_global_step() < max_iter:
iter_elapsed += 1
model.update_global_step()
data_dict = get_data(dataloader_train,
mode="train",
dataloader_itr=dataitr_train)
data = data_dict["data"]
dataitr_train = data_dict["dataloader_itr"]
train_info = train_one_iter(model, data, optimizer, lr_scheduler,
model.get_global_step())
if model.get_global_step(
) % num_save_iter == 0 or model.get_global_step() >= max_iter:
Network.save_weight(model._model, g_save_dir,
model.get_global_step())
if model._sub_model is not None:
Network.save_weight(model._sub_model, g_save_dir,
model.get_global_step())
if model.get_global_step(
) % num_log_iter == 0 or model.get_global_step() >= max_iter:
log_train_info(train_info, model.get_global_step())
time_elapsed = time.time() - g_since
ert = (time_elapsed / iter_elapsed *
(max_iter - model.get_global_step()))
print(
f"Estimated time remaining: {int(ert / 60):d} min {int(ert % 60):d} s"
)
if model.get_global_step(
) % num_val_iter == 0 or model.get_global_step() >= max_iter:
val_info = val_one_epoch(model,
dataloader_val,
x_range=(cfg.TASK["valid_range"][0],
cfg.TASK["valid_range"][3]),
y_range=(cfg.TASK["valid_range"][1],
cfg.TASK["valid_range"][4]))
log_val_info(val_info,
model.get_global_step(),
vis_param_dict={
"data_dir":
cfg.VALDATA["@root_path"],
"x_range": (cfg.TASK["valid_range"][0],
cfg.TASK["valid_range"][3]),
"y_range": (cfg.TASK["valid_range"][1],
cfg.TASK["valid_range"][4]),
"grid_size": (0.1, 0.1),
"dataset":
dataloader_val.dataset
})
Logger.log_txt("Training DONE!")
def generate_pseudo_annotation(cfg):
if "pseudo_annotation" not in cfg.TRAINDATA.keys():
return
Logger.log_txt("==========Generate Pseudo Annotations START=========")
# build voxelizer and target_assigner
voxelizer = build_voxelizer(cfg.VOXELIZER)
param = deepcopy(cfg.TARGETASSIGNER)
param["@classes"] = cfg.NETWORK["@classes_source"]
target_assigner = build_target_assigner(param)
# create pseudo dataset
## build network with evaluation mode
## do not change cfg.NETWORK
param = deepcopy(cfg.NETWORK)
## modify network._model config and make it same to network._sub_model
param["@classes_target"] = param["@classes_source"]
param["@model_resume_dict"] = param["@sub_model_resume_dict"]
param["@is_training"] = False
param["@box_coder"] = target_assigner.box_coder
param["@middle_layer_dict"]["@output_shape"] = [
1
] + voxelizer.grid_size[::-1].tolist() + [16]
param = {proc_param(k): v for k, v in param.items() if is_param(k)}
network = Network(**param).cuda()
## build dataloader without data augmentation, without shuffle, batch_size 1
param = deepcopy(cfg.TRAINDATA)
param["prep"]["@augment_dict"] = None
param["training"] = False
param["prep"]["@training"] = False
param["batch_size"] = 1
param["num_of_workers"] = 1
param["@class_names"] = cfg.NETWORK["@classes_source"]
# The following line does not affect actually.
param["prep"]["@filter_label_dict"]["keep_classes"] = cfg.NETWORK[
"@classes_source"]
dataloader_train = build_dataloader(data_cfg=param,
ext_dict={
"voxelizer":
voxelizer,
"target_assigner":
target_assigner,
"feature_map_size":
param["feature_map_size"]
})
## create new labels
### setup tmp dirs: tmp_root
data_root_path = cfg.TRAINDATA["@root_path"]
data_pc_path = os.path.join(data_root_path, "velodyne")
data_calib_path = os.path.join(data_root_path, "calib")
data_label_path = os.path.join(data_root_path, "label_2")
tmp_root_path = f"/tmp/incdet3-{time.time()}/training"
tmp_pc_path = os.path.join(tmp_root_path, "velodyne")
tmp_calib_path = os.path.join(tmp_root_path, "calib")
tmp_det_path = os.path.join(tmp_root_path, "detections")
tmp_label_path = os.path.join(tmp_root_path, "label_2")
tmp_splitidx_path = os.path.join(os.path.dirname(tmp_root_path),
"split_index")
os.makedirs(tmp_root_path, exist_ok=False)
os.makedirs(tmp_label_path, exist_ok=False)
os.makedirs(tmp_splitidx_path, exist_ok=False)
### soft link lidar dir, calib dir to tmp_root dir
os.symlink(data_pc_path, tmp_pc_path)
os.symlink(data_calib_path, tmp_calib_path)
### forward model on dataloader and save detections in tmp_root dir
network.eval()
detections = []
tags = [itm["tag"] for itm in dataloader_train.dataset._kitti_infos]
calibs = [itm["calib"] for itm in dataloader_train.dataset._kitti_infos]
write_txt(sorted(tags), os.path.join(tmp_splitidx_path, "train.txt"))
for data in tqdm(dataloader_train):
data = example_convert_to_torch(data)
detection = network(data)
detections.append(detection[0])
dataset_type = str(type(dataloader_train.dataset))
dataloader_train.dataset.save_detections(detections, tags, calibs,
tmp_det_path)
### ensemble the detections and gt labels
ensemble_pseudo_anno_and_gt(gt_label_dir=data_label_path,
detection_dir=tmp_det_path,
old_classes=cfg.NETWORK["@classes_source"],
pseudo_anno_dir=tmp_label_path)
## create new info pkls
### create info pkl by system call
assert (
cfg.TRAINDATA["dataset"] == "kitti",
"We currently only support the kitti dataset for pseudo annotation.")
cmd = "python3 tools/create_data_pseudo_anno.py "
cmd += "--dataset kitti "
cmd += f"--data-dir {os.path.dirname(tmp_root_path)}"
os.system(cmd)
# update cfg.TRAINDATA
### update @root_path, @info_path
cfg.TRAINDATA["@root_path"] = tmp_root_path
cfg.TRAINDATA["@info_path"] = os.path.join(os.path.dirname(tmp_root_path),
"KITTI_infos_train.pkl")
### update classes_to_exclude
cfg.TRAINDATA["prep"]["@classes_to_exclude"] = []
Logger.log_txt("==========Generate Pseudo Annotations END=========")
def setup_cores(cfg, mode):
global g_use_fp16
if mode == "train":
# build dataloader_train
generate_pseudo_annotation(cfg)
Logger.log_txt(
"After generating the pseudo annotation, the cfg.TRAINDATA is ")
Logger.log_txt(cfg.TRAINDATA)
Logger.log_txt(
"After generating the pseudo annotation, the cfg.NETWORK is ")
Logger.log_txt(cfg.NETWORK)
voxelizer = build_voxelizer(cfg.VOXELIZER)
target_assigner = build_target_assigner(cfg.TARGETASSIGNER)
dataloader_train = build_dataloader(
data_cfg=cfg.TRAINDATA,
ext_dict={
"voxelizer": voxelizer,
"target_assigner": target_assigner,
"feature_map_size": cfg.TRAINDATA["feature_map_size"]
})
# build dataloader_val
dataloader_val = build_dataloader(data_cfg=cfg.VALDATA,
ext_dict={
"voxelizer":
voxelizer,
"target_assigner":
target_assigner,
"feature_map_size":
cfg.VALDATA["feature_map_size"]
})
# build dataloader_test
dataloader_test = None
# build model
param = cfg.NETWORK
param["@middle_layer_dict"]["@output_shape"] = [
1
] + voxelizer.grid_size[::-1].tolist() + [16]
param["@is_training"] = True
param["@box_coder"] = target_assigner.box_coder
param = {proc_param(k): v for k, v in param.items() if is_param(k)}
network = Network(**param).cuda()
# build optimizer & lr_scheduler
optimizer, lr_scheduler = build_optimizer_and_lr_scheduler(
net=network,
optimizer_cfg=cfg.TRAIN["optimizer_dict"],
lr_scheduler_cfg=cfg.TRAIN["lr_scheduler_dict"],
start_iter=network.get_global_step())
# handle fp16 training
use_fp16 = cfg.TASK["use_fp16"] if "use_fp16" in cfg.TASK.keys(
) else False
if use_fp16:
network, optimizer = amp.initialize(network,
optimizer,
opt_level="O2")
g_use_fp16 = use_fp16
elif mode == "test":
# build dataloader_train
voxelizer = build_voxelizer(cfg.VOXELIZER)
target_assigner = build_target_assigner(cfg.TARGETASSIGNER)
dataloader_train = None
# build dataloader_val
dataloader_val = None
# build dataloader_test
dataloader_test = build_dataloader(data_cfg=cfg.TESTDATA,
ext_dict={
"voxelizer":
voxelizer,
"target_assigner":
target_assigner,
"feature_map_size":
cfg.TESTDATA["feature_map_size"]
})
# build model
param = cfg.NETWORK
param["@is_training"] = False
param["@box_coder"] = target_assigner.box_coder
param["@middle_layer_dict"]["@output_shape"] = [
1
] + voxelizer.grid_size[::-1].tolist() + [16]
param = {proc_param(k): v for k, v in param.items() if is_param(k)}
network = Network(**param).cuda()
# build optimizer & lr_scheduler
optimizer, lr_scheduler = None, None
elif mode in ["compute_ewc_weights", "compute_mas_weights"]:
voxelizer = build_voxelizer(cfg.VOXELIZER)
target_assigner = build_target_assigner(cfg.TARGETASSIGNER)
dataloader_train = build_dataloader(
data_cfg=cfg.TRAINDATA,
ext_dict={
"voxelizer": voxelizer,
"target_assigner": target_assigner,
"feature_map_size": cfg.TRAINDATA["feature_map_size"]
})
dataloader_val, dataloader_test = None, None
# build model
param = cfg.NETWORK
param["@middle_layer_dict"]["@output_shape"] = [
1
] + voxelizer.grid_size[::-1].tolist() + [16]
param["@is_training"] = True
param["@box_coder"] = target_assigner.box_coder
param = {proc_param(k): v for k, v in param.items() if is_param(k)}
network = Network(**param).cuda()
# build optimizer & lr_scheduler
optimizer, lr_scheduler = None, None
else:
raise NotImplementedError
cores = {
"dataloader_train": dataloader_train,
"dataloader_val": dataloader_val,
"dataloader_test": dataloader_test,
"model": network,
"optimizer": optimizer,
"lr_scheduler": lr_scheduler
}
return cores