def rot_box(box,cterxy,imgwh,rot_angle,scale=1.0,correction=True):
'''
Args:
box:边框坐标[xmin,ymin,xmax,ymax]
cterxy:旋转中心点坐标 [cter_x,cter_y]
imgwh:图片宽高[w,h]
rot_angle:旋转角
scale:放缩尺度
correction: bool,修正旋转后的目标框为正常左上右下坐标
return:
box:边框坐标[x1,y1,x2,y2,x3,y3,x4,y4]，左上开始，逆时针
'''
result_box = []
xmin,ymin,xmax,ymax = box[0],box[1],box[2],box[3]
complete_coords = [xmin,ymin,xmin,ymax,xmax,ymax,xmax,ymin]
for i in range(int(len(complete_coords)/2)):
rotx,roty = rot_xy(cterxy[0],cterxy[1],complete_coords[2*i],complete_coords[2*i+1],rot_angle,scale)
result_box.append(rotx)
result_box.append(roty)
if correction:
xmin = min(result_box[0:len(result_box):2])
xmax = max(result_box[0:len(result_box):2])
ymin = min(result_box[1:len(result_box):2])
ymax = max(result_box[1:len(result_box):2])
xmin_v = utils.confine(xmin,0,imgwh[0]-1)
ymin_v = utils.confine(ymin,0,imgwh[1]-1)
xmax_v = utils.confine(xmax,0,imgwh[0]-1)
ymax_v = utils.confine(ymax,0,imgwh[1]-1)
#使用阈值剔除边缘截断严重的目标
if utils.calc_iou([xmin,ymin,xmax,ymax],[xmin_v,ymin_v,xmax_v,ymax_v]) < 0.5:
xmin_v,ymin_v,xmax_v,ymin_v=0,0,0,0
return [xmin_v,ymin_v,xmin_v,ymax_v,xmax_v,ymax_v,xmax_v,ymin_v]
else:
return complete_coords
def sample_area(self, pred_box1, num, thre_max=0.5, thre_min=0):
# sample area
pred_box = pred_box1.squeeze()
pred_box_w = pred_box[2] - pred_box[0]
pred_box_h = pred_box[3] - pred_box[1]
pred_box_cx = pred_box[0] + pred_box_w / 2.0
pred_box_cy = pred_box[1] + pred_box_h / 2.0
samples = np.ones((num, 4), dtype=np.float32)
samples_wh = np.ones((num, 2), dtype=np.float32)
# samples_wh[:,0]*=pred_box_w
# samples_wh[:,1]*=pred_box_h
cur_n = 0
# s:(0.3,0.7664)
s = 0.23 + np.random.chisquare(3, num) * 0.09
s = 1 - s
ind = np.where(s > 0.3)[0]
s = s[ind[:num / 2]]
samples_wh[:num / 2, 0] = pred_box_w * s
samples_wh[:num / 2, 1] = pred_box_h * s
# s:(1.3258,2)
s = 1.325 + np.random.chisquare(3, num) * 0.125
ind = np.where(s <= 2)[0]
s = s[ind[:num / 2]]
samples_wh[num / 2:, 0] = pred_box_w * s
samples_wh[num / 2:, 1] = pred_box_h * s
# (x_c,y_c)-->(x1,y1,x2,y2)
samples[:, 0] = pred_box_cx - samples_wh[:, 0] / 2.0
samples[:, 1] = pred_box_cy - samples_wh[:, 1] / 2.0
samples[:, 2] = pred_box_cx + samples_wh[:, 0] / 2.0
samples[:, 3] = pred_box_cy + samples_wh[:, 1] / 2.0
samples = utils.restrict_box(samples, self.width, self.height)
iou = utils.calc_iou(pred_box, samples)
iou = iou[:, np.newaxis]
return samples, iou
def nms_box(preds, confm, feat, conf):
# preds:(N,4),(x1,y1,x2,y2)
# N=preds.shape[0]
res_box = []
res_feat = []
res_conf = []
thresh = 0.5 # 0.7
t = []
while preds.shape[0] > 0:
ind = np.argsort(-confm.squeeze()).tolist()
i = ind[0]
cur_box = preds[i]
cur_iou = utils.calc_iou(cur_box, preds)
i_spa = np.where(cur_iou < thresh)[0]
res_box.append(cur_box)
res_feat.append(feat[i])
res_conf.append(conf[i, :])
t.append(preds.shape[0] - i_spa.shape[0])
preds = preds[i_spa]
confm = confm[i_spa]
feat = feat[i_spa]
conf = conf[i_spa]
return np.array(res_box), np.array(res_feat), np.array(res_conf), np.array(
t)
def toi_labeling(
self,
toi_boxes,
entities,
mode,
):
ious = calc_iou(toi_boxes, np.array([(e[0], e[1]) for e in entities]))
max_ious = ious.max(axis=1)
max_idx = ious.argmax(axis=1)
tois = []
for i in range(toi_boxes.shape[0]):
if max_ious[i] == 1:
self.candidation_hit[mode] += 1
tois.append(
(toi_boxes[i, 0], toi_boxes[i,
1], entities[max_idx[i]][2]))
elif max_ious[i] >= 1 and mode == "train":
tois.append(
(toi_boxes[i, 0], toi_boxes[i,
1], entities[max_idx[i]][2]))
elif max_ious[i] < self.config.train_neg_iou_th or mode != "train":
tois.append((toi_boxes[i, 0], toi_boxes[i, 1], 0))
self.tois_num[mode] += (len(tois))
return sorted(tois)
def generate_patch(boxes, threshold):
""" Function to generate a random patch within the image
If the patch overlaps any gt boxes at above the threshold,
then the patch is picked, otherwise generate another patch
Args:
boxes: box tensor (num_boxes, 4)
threshold: iou threshold to decide whether to choose the patch
Returns:
patch: the picked patch
ious: an array to store IOUs of the patch and all gt boxes
"""
while True:
patch_w = random.uniform(0.1, 1)
scale = random.uniform(0.5, 2)
patch_h = patch_w * scale
patch_xmin = random.uniform(0, 1 - patch_w)
patch_ymin = random.uniform(0, 1 - patch_h)
patch_xmax = patch_xmin + patch_w
patch_ymax = patch_ymin + patch_h
patch = np.array([[patch_xmin, patch_ymin, patch_xmax, patch_ymax]],
dtype=np.float32)
patch = np.clip(patch, 0.0, 1.0)
ious = calc_iou(tf.constant(patch), boxes)
if tf.math.reduce_any(ious >= threshold):
break
return patch[0], ious[0]
def counter_for_actual_accuracy(pred_labels,
target_labels): #find position for eval_actual
#given list of targets and predictions, find which prediction corresponds to which target.
iou_choice_counter = 0
mega_iou_choice = []
for key, pred_label_list in pred_labels.items():
if key in target_labels:
iou_list = []
target_label_list = target_labels[key]
for target_idx, target_label in enumerate(target_label_list):
for pred_idx, pred_label in enumerate(pred_label_list):
iou_val = utils.calc_iou(pred_label, target_label)
iou_list.append([
iou_val, pred_idx, target_idx, pred_label, target_label
])
list_len = min(len(target_label_list), len(pred_label_list))
iou_list = sorted(iou_list,
key=lambda k: (k[0], random.random()),
reverse=True)
iou_choice = []
while len(iou_list) != 0 and len(iou_choice) < list_len:
if len(iou_choice) == 0:
iou_choice.append(iou_list.pop(0))
else:
#pdb.set_trace()
cur_item = iou_list.pop(0)
flag = True
for item in iou_choice:
if cur_item[1] == item[1]:
flag = False
break
if cur_item[2] == item[2]:
flag = False
break
if flag:
iou_choice.append(cur_item)
mega_iou_choice.extend(iou_choice)
#============================================================================================
#for actual accuracy: check if center of prediction is within (start, end) boundaries of target
for item in mega_iou_choice:
iou_val, pred_idx, target_idx, pred_label, target_label = item
pred_start, pred_end = pred_label
target_start, target_end = target_label
center_pred = float(pred_end + pred_start) / 2
if round(center_pred, 2) >= round(target_start, 2) and round(
center_pred, 2) <= round(target_end, 2):
iou_choice_counter += 1
return iou_choice_counter
def form_gt(self, gt_bboxes: List[torch.Tensor]) -> Tuple[torch.Tensor, torch.Tensor]:
"""
Builds 4D torch.Tensor with a shape of the detector output for the batch of frames.
:param gt_bboxes: list of torch.Tensors of shape [n_bboxes, n_bbox_params]
of ground truth bounding boxes parameters which are center coordinates, length, width, sin(a) and cos(a)
:return: Tuple of two 4D torch.Tensor of ground truth data:
First tensor is of shape [batch_size, detector_out_width, detector_out_length, n_predefined_boxes]
and represents classification target with ones for boxes with associated GT boxes
Second tensor is of shape
[batch_size, detector_out_width, detector_out_length, n_predefined_boxes, n_bbox_params]
and represents regression target
"""
gt_bboxes_coords = [utils.bbox_to_coordinates(gt_bbox, rot=False).to(self.device) for gt_bbox in gt_bboxes]
classification_target = []
regression_target = []
prior_boxes_params = self.prior_boxes_params.view(-1, self.n_bbox_params)
prior_boxes_coords = self.prior_boxes_coords.view(-1, 4, 2) # each box is 4 points of 2 coordinates)
for gt_bbox, gt_bbox_coords in zip(gt_bboxes, gt_bboxes_coords):
iou = utils.calc_iou(gt_bbox_coords, prior_boxes_coords) # tensor of size [n_gt_boxes, n_prior_boxes]
gt_has_match = (iou >= self.iou_threshold).sum(dim=1).bool()
prior_has_match = (iou >= self.iou_threshold).sum(dim=0).bool()
gt_best_match = iou.argmax(dim=1)
prior_match = iou.argmax(dim=0)
prior_match[gt_best_match[~gt_has_match]] = (~gt_has_match).nonzero().flatten()
prior_has_match[gt_best_match[~gt_has_match]] = True
prior_has_match = prior_has_match.view(
self.detector_out_width, self.detector_out_length, len(self.predefined_bboxes)
).float()
cur_regression_target = torch.zeros(self.detector_out_width * self.detector_out_length *
len(self.predefined_bboxes), self.n_bbox_params, device=self.device)
# Calculate regression target: y and x offset, width and length correction, sin and cos of angle
cur_regression_target[:, 0] = \
(prior_boxes_params[:, 0] - gt_bbox[prior_match, 0]) / gt_bbox[prior_match, 2]
cur_regression_target[:, 1] = \
(prior_boxes_params[:, 1] - gt_bbox[prior_match, 1]) / gt_bbox[prior_match, 3]
cur_regression_target[:, 2] = torch.log(prior_boxes_params[:, 2] / gt_bbox[prior_match, 2])
cur_regression_target[:, 3] = torch.log(prior_boxes_params[:, 3] / gt_bbox[prior_match, 3])
cur_regression_target[:, 4] = gt_bbox[prior_match, 4]
cur_regression_target[:, 5] = gt_bbox[prior_match, 5]
cur_regression_target = \
cur_regression_target.view(self.detector_out_width, self.detector_out_length,
len(self.predefined_bboxes), self.n_bbox_params)
cur_regression_target *= prior_has_match.unsqueeze(3)
regression_target.append(cur_regression_target)
classification_target.append(prior_has_match)
return torch.stack(classification_target), torch.stack(regression_target)
def sample_xy(self, pred_box1, num, thre_max=0.5, thre_min=0.0):
# only perform xy shift
# x alone
pred_box = pred_box1.squeeze()
alone = 1 / 3.0
both = 1 - np.sqrt(2 / 3.0)
pred_box_w = pred_box[2] - pred_box[0]
pred_box_h = pred_box[3] - pred_box[1]
pred_box_cx = pred_box[0] + pred_box_w / 2.0
pred_box_cy = pred_box[1] + pred_box_h / 2.0
samples = np.ones((num, 4), dtype=np.float32)
samples_c = np.ones((num, 2), dtype=np.float32)
samples_c[:, 0] *= pred_box_cx
samples_c[:, 1] *= pred_box_cy
num1 = num / 2
# +-dx,+-dy
dx = 0.25 + np.random.chisquare(3, num1) * 0.1
samples_c[:num1 / 4, 0] += (pred_box_w * dx[:num1 / 4]) # dx
samples_c[num1 / 4:num1 / 2, 0] -= (pred_box_w * dx[num1 / 4:num1 / 2]) # -dx
samples_c[num1 / 2:3 * num1 / 4, 1] += (pred_box_h * dx[num1 / 2:num1 * 3 / 4]) # dy
samples_c[3 * num1 / 4:num1, 1] -= (pred_box_h * dx[3 * num1 / 4:]) # -dy
# |dx2|=|dy2|
dx2 = 0.15 + np.random.chisquare(3, num) * 0.03
# dx,dy
samples_c[num1:(num1 + num1 / 4), 0] += (pred_box_w * dx2[:num1 / 4])
samples_c[num1:(num1 + num1 / 4), 1] += (pred_box_h * (dx2[num1 / 4:num1 / 2]))
# -dx,dy
samples_c[(num1 + num1 / 4):(num1 + num1 / 2), 0] -= (pred_box_w * dx2[num1 / 2:num1 * 3 / 4])
samples_c[(num1 + num1 / 4):(num1 + num1 / 2), 1] += (pred_box_h * (dx2[3 * num1 / 4:num1]))
# dx,-dy
samples_c[(num1 + num1 / 2):(num1 + 3 * num1 / 4), 0] += (pred_box_w * dx2[num1:(num1 + num1 / 4)])
samples_c[(num1 + num1 / 2):(num1 + 3 * num1 / 4), 1] -= (
pred_box_h * (dx2[(num1 + num1 / 4):(num1 + num1 / 2)]))
# -dx,-dy
samples_c[(num1 + 3 * num1 / 4):, 0] -= (pred_box_w * dx2[(num1 + num1 / 2):(num1 + 3 * num1 / 4)])
samples_c[(num1 + 3 * num1 / 4):, 1] -= (
pred_box_h * (dx2[(num1 + 3 * num1 / 4):]))
# (x_c,y_c)-->(x1,y1,x2,y2)
samples[:, 0] = samples_c[:, 0] - pred_box_w / 2.0
samples[:, 1] = samples_c[:, 1] - pred_box_h / 2.0
samples[:, 2] = samples_c[:, 0] + pred_box_w / 2.0
samples[:, 3] = samples_c[:, 1] + pred_box_h / 2.0
samples = utils.restrict_box(samples, self.width, self.height)
iou = utils.calc_iou(pred_box, samples)
iou = iou[:, np.newaxis]
return samples, iou
def nms_pred(pred,preds, vpca3,conf):
# preds:(N,4),(x1,y1,x2,y2)
# N=preds.shape[0]
thresh = 0.3 # 0.7
cur_iou = utils.calc_iou(pred, preds)
i_spa = np.where(cur_iou < thresh)[0]
i_spa2 = np.where(conf > 0.4)[0]
print "IOU: ",i_spa
print "Conf: ",i_spa2
ind = np.array(list(set(i_spa).intersection(set(i_spa2))))
print ind
if ind.shape[0] > 0:
res_box = preds[ind,:]
res_vpca3 = vpca3[ind,:]
return True,res_vpca3
else:
return False,np.zeros((1,4))
def crop_box(box, top_left_x, top_left_y, crop_w, crop_h, iou_thr=0.5):
'''目标框坐标平移变换
Args:
box:目标框坐标[xmin,ymin,xmax,ymax]
top_left_x,top_left_y:裁剪图像左上角坐标
crop_w,crop_h:裁剪部分图像宽高
iou_thr: iou阈值,去除裁剪后过小目标
return:
crop_box:平移变换结果[xmin,ymin,xmax,ymax]
'''
xmin, ymin = crop_xy(box[0], box[1], top_left_x, top_left_y, crop_w,
crop_h)
xmax, ymax = crop_xy(box[2], box[3], top_left_x, top_left_y, crop_w,
crop_h)
croped_box = [xmin, ymin, xmax, ymax]
if utils.calc_iou([0, 0, box[2] - box[0], box[3] - box[1]],
[0, 0, xmax - xmin, ymax - ymin]) < iou_thr:
croped_box = [0, 0, 0, 0]
return croped_box
def evaluate_test_dataset(device, exp, dataset, N=5):
print(f'******* {dataset}')
miou = []
path = os.path.join(opt.dataroot, dataset, 'experiments', exp)
model_type = 101 if 'ResNet101' in exp else 50
for i in range(N):
model = load_model(device, model_type, path, i)
test_loader = load_test_dataset()
iou = []
i = 0
for X, y in test_loader:
X = X.to(device)
y = y.to(device)
output = model(X)['out']
probs = torch.functional.F.softmax(output, 1)
y_pred = torch.argmax(probs, dim=1)
iou.append(calc_iou(y_pred, y))
i += 1
if i == 76: break
miou.append(np.mean(iou))
print(miou[-1])
print(miou)
print(np.mean(miou), np.std(miou))
def train():
assert max_point == 8192, "wrong max point,need to change pointnet_util->fp1 = PointNetFeaturePropagation_PointConv->in_channel"
# read device
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print("device: {}".format(device), end="\n")
# load dataset
data_loader_train = Scannetloader(scannet_file_root,
max_point,
n_class,
is_train=True)
data_loader_val = Scannetloader(scannet_file_root,
max_point,
n_class,
is_train=False)
train_loader = torch.utils.data.DataLoader(data_loader_train,
batch_size=batch_size,
shuffle=True,
num_workers=n_workers)
val_loader = torch.utils.data.DataLoader(data_loader_val,
batch_size=batch_size,
shuffle=True,
num_workers=n_workers)
# set model running devices
#model = PointNet2SemSeg(n_class)
model = PointNet2SemSeg_PointConv(n_class)
model.to(device)
#?torch.backends.cudnn.benchmark = True
print("usable gpu nums: {}".format(torch.cuda.device_count()))
# set optimizer, lr_scheduler, loss function
optimizer = None
if optimizer_name == "rmsprop":
optimizer = torch.optim.RMSprop(model.parameters(),
lr=lr,
momentum=momentum,
weight_decay=decay_rate)
elif optimizer_name == "sgd":
optimizer = torch.optim.SGD(model.parameters(),
lr=lr,
momentum=momentum,
weight_decay=decay_rate)
elif optimizer_name == "adam":
optimizer = torch.optim.Adam(model.parameters(),
lr=lr,
betas=(0.9, 0.999),
eps=1e-08)
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=40,
gamma=0.5)
# criterion
#criterion = torch.nn.BCEWithLogitsLoss()
#criterion = F.nll_loss()
model, optimizer = amp.initialize(model, optimizer,
opt_level="O1") # O1 not 01
model = torch.nn.DataParallel(
model, device_ids=[i for i in range(torch.cuda.device_count())])
# load checkpoints
last_best_iou = -100.
load_ckpt_path = os.path.join("checkpoints", load_ckpt_name)
start_epoch = 0
if is_load_checkpoints:
if torch.cuda.is_available():
checkpoint = torch.load(load_ckpt_path)
else:
checkpoint = torch.load(load_ckpt_path, map_location='cpu')
model.load_state_dict(checkpoint['model_state'])
last_best_iou = checkpoint['best_iou']
start_epoch = checkpoint['epoch']
optimizer.load_state_dict(checkpoint["optimizer"])
amp.load_state_dict(checkpoint["amp"])
print('Checkpoint resume success... last iou:{:.4%},last epoch:{}'.
format(last_best_iou, start_epoch))
# train epoch
best_iou = last_best_iou
time_epoch = time.time()
i = 0
print("training...")
for epoch in range(start_epoch, max_epoch):
time_step = time.time()
lr_scheduler.step(epoch=epoch)
for step, batch in enumerate(train_loader):
model.train()
points = batch[0].to(device)
targets = batch[1].long().to(device)
points = points.transpose(2, 1)
optimizer.zero_grad()
# print(points.shape) = [3, 9, 8192]
pred = model(points[:, :3, :], points[:, 3:, :])
pred = pred.contiguous().view(-1, n_class)
# targets = targets.view(-1, 1)[:, 0]
targets = targets.contiguous().view(-1)
#loss = criterion(pred.float(),targets.float())
loss = F.cross_entropy(input=pred,
target=targets) # log_softmax + nll_loss
# loss = F.nll_loss(pred, targets) # softmax + Cross-Entropy cost
cur_loss = np.float(loss.cpu().detach().numpy(
)) # 不要用.data,会导致tensor flag: requires_grad=False,终止求导
with amp.scale_loss(loss, optimizer) as scaled_loss:
scaled_loss.backward()
#loss.backward()
optimizer.step()
writer.add_scalars('Train_record/Loss per X step',
{'loss': cur_loss},
step + epoch * len(train_loader))
cur_time = time.time()
print(
"\rtrain | epoch: {}/{} | step:{}/{} | {} s/step | time used:{:d}s | currency loss:{:.4}"
.format(epoch, max_epoch, step + 1, len(train_loader),
int((cur_time - time_step) / (step + 1)),
int(cur_time - time_epoch), cur_loss),
end='',
flush=True)
# val
print("\nvaluing")
with torch.no_grad():
model.eval()
cat_table = 0
total_cat_table = np.zeros((2, n_class)) # [0]:acc,[1]:miou
time_step_val = time.time()
for step_val, batch_val in enumerate(val_loader):
#print("read batch time used:",time.time()-time_step)
points_val = batch_val[0].to(device)
labels_val = batch_val[1].cpu().data.numpy()
points_val = points_val.transpose(2, 1)
pred_val = model(points_val[:, :3, :], points_val[:, 3:, :])
# pred_val.shape = [batch_size,n_point,n_class]
pred_val = F.softmax(pred_val, dim=2).cpu().data.numpy()
pred_val = np.argmax(pred_val, axis=2)
#visualizer(points_val.transpose(2,1)[0],pred_val[0],labels_val[0])
cat_table = calc_iou(pred_val, labels_val)
total_cat_table += cat_table
cur_time = time.time()
print(
"\rvalue | step:{}/{} | {} s/step | mean iou:{:.4} | mean acc:{:.4}"
.format(step_val + 1, len(val_loader),
int((cur_time - time_step_val) / (step_val + 1)),
np.mean(cat_table[1]), np.mean(cat_table[0])),
end='',
flush=True)
total_cat_table = total_cat_table / len(val_loader)
# total_cat_table = total_cat_table / (step_val+1)
mean_iou = np.mean(total_cat_table[1])
mean_acc = np.mean(total_cat_table[0])
print("\ntrue accuracy:{:.3f}".format(
calc_acc(pred_val, labels_val)))
print("\naccuracy: {:.3f} | mean iou: {:.3f}".format(
mean_acc, mean_iou))
writer.add_scalars('Val_record/mIoU', {'accuracy': mean_acc},
epoch)
writer.add_scalars('val_record/mIoU', {'mIoU': mean_iou}, epoch)
print("class name \tmean accuracy\tmIoU")
for i in range(n_class):
print("%-18s %.3f \t%.3f" %
(CLASS_LABELS[i], total_cat_table[0][i],
total_cat_table[1][i]))
if mean_iou > best_iou:
best_iou = mean_iou
state = {
"epoch": epoch + 1,
"model_state": model.state_dict(),
"best_iou": best_iou,
"amp": amp.state_dict(),
"optimizer": optimizer.state_dict()
}
if not os.path.isdir('./checkpoints'):
os.mkdir('./checkpoints')
save_path = os.path.join(
'./checkpoints', "apex_eph_{}_iou_{:.2%}.ckpt.pth".format(
epoch + 1, best_iou))
torch.save(state, save_path)
print("checkpoint saved success")
writer.close()
def sample_iou_new(self, gt_box, Q, T, R, N, thre_min=0, thre_max=1):
sample_boxN = []
sample_iouN = []
cur_n = 0
D = np.array([[self.box_w, 0], [0, self.box_h]])
QL = D * Q
# QL = np.linalg.cholesky(Q)
a = 1.5 # 1.5
gt_state = utils.bbox_to_states(gt_box, self.area, self.ratio)
sample_times = 0
chg_i = 0
while cur_n < N:
sample_particles = np.zeros((N, 6), dtype=np.float32)
QL = D * Q
sample_particles[:, :2] = gt_state[:, :2] + np.dot(QL,
np.random.randn(2, N)).transpose()
if self.mltply:
dsn = np.random.randn(N) * T
ds = np.power(a, dsn)
else:
ds = 1 + np.random.randn(N) * T
ds = np.maximum(0.01, ds) # in case of ds<0
sample_particles[:, 2] = gt_state[:, 2] * ds
if self.mltply:
dr = np.random.randn(N) * R + 1
else:
dr = 1 + np.random.randn(N) * R
dr = np.maximum(0.01, dr) # in case of dr<0
sample_particles[:, 3] = gt_state[:, 3] * dr
# get box
if self.mltply:
sample_box = utils.state_to_bbox_m(sample_particles, self.area, self.ratio)
else:
sample_box = utils.state_to_bbox(sample_particles, self.area, self.ratio)
sample_box = utils.restrict_box(sample_box, self.width, self.height)
# compute iou
sample_iou = utils.calc_iou(gt_box, sample_box)
# restrict iou
ind = np.where((sample_iou >= thre_min) & (sample_iou <= thre_max))
sample_box = sample_box[ind[0]]
sample_iou = sample_iou[ind[0]]
cur_n += sample_box.shape[0]
sample_boxN.append(sample_box)
sample_iouN.append(sample_iou.reshape((-1, 1)))
if ind[0].shape[0] < N / 2 and thre_max >= 0.8:
if chg_i == 0:
Q *= 0.5
chg_i = (chg_i + 1) % 3
else:
if chg_i == 1:
T *= 0.5
T = np.minimum(T, 0.5)
chg_i = (chg_i + 1) % 3
else:
R *= 0.5
R = np.minimum(R, 0.5)
chg_i = (chg_i + 1) % 3
if ind[0].shape[0] < N / 2 and thre_min <= 0.5:
if chg_i == 0:
Q *= 2
chg_i = (chg_i + 1) % 3
else:
if chg_i == 1:
T *= 2
T = np.minimum(T, 0.5)
chg_i = (chg_i + 1) % 3
else:
R *= 2
R = np.minimum(R, 0.5)
chg_i = (chg_i + 1) % 3
sample_times += 1
if sample_times >= 100: # and cur_n>N/2.0:#100
# print "Caution: too many loops in sampling"
# break
raise OverflowError()
if cur_n >= N:
sample_boxN = np.vstack(sample_boxN)[:N, :]
sample_iouN = np.vstack(sample_iouN)[:N, :]
else:
diff_n = N - cur_n
sample_iouN = np.vstack(sample_iouN)
sample_boxN = np.vstack(sample_boxN)
diff_ind = random.sample(range(cur_n), diff_n) # need to ensure diff_n<cur_n
sample_boxN = np.vstack([sample_boxN, sample_boxN[diff_ind]])
sample_iouN = np.vstack([sample_iouN, sample_iouN[diff_ind]])
return sample_boxN, sample_iouN
for i in range(filter.num_particles):
cx = filter.particles[i, 0]
cy = filter.particles[i, 1]
cv2.circle(frame_data, (int(cx), int(cy)), 1, (0, 0, 255), thickness=1)
# compute conf
conf = np.zeros(filter.weights.shape)
# np.save('particles.npy',filter.particles)
pred_boxes = utils.state_to_bbox(filter.particles, area, ratio)
# print 'pred_boxes: ',pred_boxes
# for i in range(conf.shape[0]):
# pred_box=pred_boxes[i,:]
# print "pred_box is: ",pred_box
# conf[i]=np.dot(gt_box,pred_box)/np.linalg.norm(gt_box,ord=2)
# conf[i]=np.dot(gt_box,pred_box)/np.sum(np.square(gt_box))
conf = utils.calc_iou(gt_box, pred_boxes)
# print 'conf is: ',conf
filter.update_particles(conf)
if filter.neff() < len(filter.particles): # 1/2
filter.resample()
pred_state = filter.estimate()
pred_box = utils.state_to_bbox(pred_state.reshape((-1, 6)), area, ratio)
print 'ground truth bbox is: ', gt_box
print "pred_box is: ", pred_box
iou = utils.calc_iou(gt_box, pred_box)
print 'iou is: ', iou
# (B,G,R)
show_frame = True
cv2.circle(frame_data, (int(filter.cur_pos[0, 0]), int(filter.cur_pos[0, 1])), 2, (0, 0, 255), thickness=1)
if show_frame:
cv2.imshow(sequence, frame_data)
#print('Begin to train model_rpn')
#test
#print(np.shape(X))
#print(np.shape(Y[0]),' ',np.shape(Y[1]))
#P_rpn = model_rpn.predict_on_batch(X)
#print(np.shape(P_rpn[0]),' ',np.shape(P_rpn[1]))
#endtest
loss_rpn = model_rpn.train_on_batch(X, Y) #训练model_rpn
#print('Predict rpn')
P_rpn = model_rpn.predict_on_batch(X) #测试rpn
#R:boxes, probs 返回经过npm后剩下的bbox以及对应的probs （（左上，右下坐标），序号） （anchors， 序号 ）
R = rpn_to_roi(P_rpn[0], P_rpn[1], use_regr=True, overlap_thresh=0.7, max_boxes=300)
# note: calc_iou converts from (x1,y1,x2,y2) to (x,y,w,h) format
#X：[[x1, y1, w, h],...] Y1：[[0,0,1,0]] ,......] Y2：[[1,1,1,1],....] , [sx*tx, sy*ty, sw*tw, sh*th],...]
X2, Y1, Y2, IouS = calc_iou(R, img_data, class_mapping )
if X2 is None:
rpn_accuracy_rpn_monitor.append(0)
rpn_accuracy_for_epoch.append(0)
continue
neg_samples = np.where(Y1[0, :, -1] == 1) #非背景 #直接对应坐标的位置
pos_samples = np.where(Y1[0, :, -1] == 0) #背景
if len(neg_samples) > 0:
neg_samples = neg_samples[0]
else:
neg_samples = []
if len(pos_samples) > 0:
def main(args):
vis = args.vis
debug = args.debug
save = args.save
nparticles = args.particles
root_path = '/home/ccjiang/Documents/caffe-fast-rcnn/examples/tracker/'
dataset_path = "/data/OTB100"
sequence = args.sequence
model_def = os.path.join(root_path, args.prototxt)
model_weight = os.path.join(root_path, args.caffemodel)
for t in os.walk(os.path.join(dataset_path, sequence, sequence, "img")):
if t[0] == os.path.join(dataset_path, sequence, sequence, "img"):
nFrame = len(t[2])
print "Total frames are: ", nFrame
gt_path = os.path.join(dataset_path, sequence, sequence,
"groundtruth_rect.txt")
gt_boxes = utils.get_boxes_all(gt_path)
vggnet = VGGnet.VGGnet(model_def, model_weight)
thre_min_neg = 0.0
thre_max_neg = 0.4 #0.5
thre_min_pos = 0.8
thre_max_pos = 1.0
conf_hist = []
iou_hist = []
area_hist = []
eig_hist = []
pred_hist = [] #(x1,y1,x2,y2)
reinit = 0
nFrame = np.minimum(nFrame, gt_boxes.shape[0])
for id in np.arange(0, nFrame): #nFrame
frame_name = "img/%04d.jpg" % (id + 1)
print "Start processing: %s" % frame_name
frame_path = os.path.join(dataset_path, sequence, sequence, frame_name)
frame_data = caffe.io.load_image(frame_path) # (432,576,3), in [0,1]
gt_box = gt_boxes[id]
if id == 0:
h, w, c = frame_data.shape
frame_shape = [c, w, h]
fps = 20
fourcc = cv2.VideoWriter_fourcc(*'MJPG')
video_writer = cv2.VideoWriter("res_%s.avi" % sequence, fourcc,
fps, (w, h))
fail_times = 0
area = (gt_box[2] - gt_box[0]) * (gt_box[3] - gt_box[1])
ratio = (gt_box[2] - gt_box[0]) / (gt_box[3] - gt_box[1]
) #ratio=w/h
# set up net.blobs['im_info']
print "Image Size: ", w, h
vggnet.reshape(w=w, h=h, nbox=nparticles)
filter = PFfilter.PFfilter(
utils.bbox_to_states(gt_box, area, ratio), area, ratio, w, h,
nparticles)
filter.create_particles()
filter.restrict_particles(w, h)
area_hist.append(filter.cur_a)
pred_hist.append(np.array(gt_box).reshape(1, -1))
#pca
# test sample_iou
num_true = 500
num_false = 1000
boxes_train = []
#boxes_train_neg=[]
iou_train = []
try:
#Q=[[1,0],[0,1]] #for pixel wise
Q = 0.05 #box_w,box_h
sample_box_true, sample_iou_true = filter.sample_iou(
gt_box, Q, 0.01, 0.01, num_true, 0.8, 1.0)
except OverflowError as e:
print "too many loops in sample."
# print sample_box_true[:10]
# print sample_box_true.shape[0]
# print sample_iou_true[:10]
print "average iou: ", np.mean(sample_iou_true)
boxes_train.append(sample_box_true)
iou_train.append(sample_iou_true)
try:
#Q=[[36,0],[0,36]]#for pixel wise
Q = 0.2 #0.15
sample_box_false, sample_iou_false = filter.sample_iou(
gt_box, Q, 0.2, 0.01, num_false / 2, 0, thre_max_neg)
except OverflowError as e:
print "too many loops in sample."
# print sample_box_false[:10]
# print sample_box_false.shape[0]
# print sample_iou_false[:10]
print "average iou: ", np.mean(sample_iou_false)
boxes_train.append(sample_box_false)
iou_train.append(sample_iou_false)
try:
#Q=[[36,0],[0,36]]#for pixel wise
Q = 0.2
sample_box_false, sample_iou_false = filter.sample_iou(
gt_box, Q, 0.01, 0.2, num_false / 2, 0, thre_max_neg)
except OverflowError as e:
print "too many loops in sample."
# print sample_box_false[:10]
# print sample_box_false.shape[0]
# print sample_iou_false[:10]
print "average iou: ", np.mean(sample_iou_false)
boxes_train.append(sample_box_false)
iou_train.append(sample_iou_false)
boxes_train = np.vstack(boxes_train)
iou_train = np.vstack(iou_train)
y_train_true = np.ones((num_true, ))
y_train_false = np.zeros((num_false, ))
y_train = np.hstack([y_train_true, y_train_false])
#permutation
ind_perm = np.random.permutation(range(num_false + num_true))
boxes_train = boxes_train[ind_perm, :]
iou_train = iou_train[ind_perm]
y_train = y_train[ind_perm]
ind_pos = np.where(y_train == 1)[0]
ind_neg = np.where(y_train == 0)[0]
vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0])
features = vggnet.get_features_first_raw(frame_data,
boxes_raw=boxes_train,
id=id)
for k, v in features.iteritems():
print k, v.shape
if k == 'f3':
pca3 = utils.skl_pca(v)
v_pca3 = pca3.transform(v)
pca3_pos = np.zeros((num_true, pca3.n_components_),
dtype=np.float32)
pca3_neg = np.zeros((num_false, pca3.n_components_),
dtype=np.float32)
pca3_pos[...] = v_pca3[ind_pos, :]
pca3_neg[...] = v_pca3[ind_neg, :]
#utils.vis_as_image(v_pca3)
#plt.imshow(v_pca3)
#plt.title("PCA features")
#plt.show()
#plt.close()
#logistic regression
y_weight = sklearn.utils.class_weight.compute_class_weight(
class_weight='balanced',
classes=np.array([0, 1]),
y=y_train)
#print y_weight
class_weight = {0: y_weight[0], 1: y_weight[1]}
clf3 = SVC(kernel="linear")
#clf3=linear_model.LogisticRegression(fit_intercept=True,solver='liblinear')
clf3.fit(v_pca3, y_train)
score3 = clf3.score(v_pca3, y_train)
print 'score3: ', score3
#prob=clf3.predict_proba(v_pca3)
print clf3.classes_
#print prob
vis_feature = False
if vis_feature:
utils.vis_features(features, id)
start_time = time.time()
else:
if fail_times >= 5:
#reinitialize
reinit += 1
area = (gt_box[2] - gt_box[0]) * (gt_box[3] - gt_box[1])
ratio = (gt_box[2] - gt_box[0]) / (gt_box[3] - gt_box[1])
filter = PFfilter.PFfilter(
utils.bbox_to_states(gt_box, area, ratio), area, ratio, w,
h, nparticles)
#filter.reset(utils.bbox_to_states(gt_box, area, ratio), area, ratio)
filter.create_particles()
filter.restrict_particles(w, h)
area_hist.append(filter.cur_a)
pred_box = gt_box
boxes_train = []
pred_hist.append(np.array(gt_box).reshape(1, -1))
#pred_hist.append(pred_box)
conf_hist.append(-0.1)
# boxes_train_neg=[]
iou_train = []
try:
# Q=[[1,0],[0,1]] #for pixel wise
Q = 0.05 # box_w,box_h
sample_box_true, sample_iou_true = filter.sample_iou(
gt_box, Q, 0.01, 0.01, num_true, 0.8, 1.0)
except OverflowError as e:
print "too many loops in sample."
# print sample_box_true[:10]
# print sample_box_true.shape[0]
# print sample_iou_true[:10]
print "average iou: ", np.mean(sample_iou_true)
boxes_train.append(sample_box_true)
iou_train.append(sample_iou_true)
try:
# Q=[[36,0],[0,36]]#for pixel wise
Q = 0.2 #0.15
sample_box_false, sample_iou_false = filter.sample_iou(
gt_box, Q, 0.2, 0.01, num_false / 2, 0, thre_max_neg)
except OverflowError as e:
print "too many loops in sample."
# print sample_box_false[:10]
# print sample_box_false.shape[0]
# print sample_iou_false[:10]
print "average iou: ", np.mean(sample_iou_false)
boxes_train.append(sample_box_false)
iou_train.append(sample_iou_false)
try:
# Q=[[36,0],[0,36]]#for pixel wise
Q = 0.2
sample_box_false, sample_iou_false = filter.sample_iou(
gt_box, Q, 0.01, 0.2, num_false / 2, 0, thre_max_neg)
except OverflowError as e:
print "too many loops in sample."
# print sample_box_false[:10]
# print sample_box_false.shape[0]
# print sample_iou_false[:10]
print "average iou: ", np.mean(sample_iou_false)
boxes_train.append(sample_box_false)
iou_train.append(sample_iou_false)
boxes_train = np.vstack(boxes_train)
iou_train = np.vstack(iou_train)
y_train_true = np.ones((num_true, ))
y_train_false = np.zeros((num_false, ))
y_train = np.hstack([y_train_true, y_train_false])
# permutation
ind_perm = np.random.permutation(range(num_false + num_true))
boxes_train = boxes_train[ind_perm, :]
iou_train = iou_train[ind_perm]
y_train = y_train[ind_perm]
ind_pos = np.where(y_train == 1)[0]
ind_neg = np.where(y_train == 0)[0]
vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0])
features = vggnet.get_features_first_raw(frame_data,
boxes_raw=boxes_train,
id=id)
for k, v in features.iteritems():
print k, v.shape
if k == 'f3':
v_pca3 = pca3.transform(v)
pca3_pos[...] = v_pca3[ind_pos, :]
pca3_neg[...] = v_pca3[ind_neg, :]
clf3.fit(v_pca3, y_train)
score3 = clf3.score(v_pca3, y_train)
print 'score3: ', score3
# prob=clf3.predict_proba(v_pca3)
print clf3.classes_
fail_times = 0
continue
filter.predict_particles(Q=0.02, cr=0.05,
ca=0.05) #0.02,0.0005,0.005
filter.restrict_particles(w, h)
area_hist.append(filter.cur_a)
#compute conf
conf = np.zeros(filter.weights.shape)
#np.save('particles.npy',filter.particles)
pred_boxes = utils.state_to_bbox(filter.particles, area, ratio)
vggnet.reshape(w, h, filter.num_particles)
features = vggnet.get_features_first_raw(frame_data,
boxes_raw=pred_boxes,
id=id)
for k, v in features.iteritems():
print k, v.shape
if k == 'f3':
v_pca3 = pca3.transform(v)
#utils.vis_as_image(v_pca3)
#plt.imshow(v_pca3)
#plt.title("PCA features")
#plt.show()
#plt.close()
#logistic regression
#conf=clf3.predict_proba(v_pca3)[:,1]
conf = -clf3.decision_function(v_pca3)
conf_max = np.max(conf)
conf_min = np.min(conf)
print 'conf_max: ', conf_max
print 'conf_min: ', conf_min
filter.update_particles(conf)
# pred_state = filter.estimate()
print filter.weights
filter.resample() # always resample
pred_state, s_particles, r_particles = filter.estimate(k=10)
cov_particles = np.dot(
filter.particles[:, :4].T,
filter.particles[:, :4]) / filter.particles.shape[0]
eigval, eigvec = np.linalg.eig(cov_particles)
max_val = eigval[0]
eig_hist.append(max_val)
print 'Max eigvalue: %f' % max_val
#print 'conf is: ',conf
if conf_max > 0 and max_val < 200000:
fail_times = 0
else:
fail_times += 1
#filter.update_particles(conf)
#pred_state=filter.estimate()
#filter.resample()
#pred_state, s_particles, r_particles = filter.estimate(k=10)
print "conf_max too low, not update particles "
pred_box = utils.state_to_bbox(pred_state.reshape((-1, 6)), area,
ratio)
print 'ground truth bbox is: ', gt_box
print "pred_box is: ", pred_box
show_sr = False
if show_sr:
plt.hist2d(s_particles,
r_particles,
bins=50,
weights=filter.weights.squeeze())
'''
plt.scatter(s_particles,r_particles,c='r',marker='.',linewidths=1)
plt.xlabel('Area')
plt.ylabel('Aspect ratio')
plt.title('Area and Ratio of particles')
plt.axis('equal')
'''
plt.show()
iou = utils.calc_iou(gt_box, pred_box)
print 'iou is: ', iou
pred_hist.append(pred_box)
conf_hist.append(conf_max)
iou_hist.append(iou)
if conf_max >= 0.1: #0.5
#update pca3_pos and pca3_neg
new_true = 100 #50
new_false = 200 #100
boxes_train = []
iou_train = []
Q = 0.02
try:
sample_box_true, sample_iou_true = filter.sample_iou(
pred_box, Q, 0.01, 0.01, new_true, 0.85, 1.0)
except OverflowError as e:
print "too many loops in sample."
# print sample_box_true[:10]
# print sample_box_true.shape[0]
# print sample_iou_true[:10]
print "average iou: ", np.mean(sample_iou_true)
boxes_train.append(sample_box_true)
iou_train.append(sample_iou_true)
try:
Q = 0.2
sample_box_false, sample_iou_false = filter.sample_iou(
pred_box, Q, 0.2, 0.01, new_false / 2, 0, thre_max_neg)
except OverflowError as e:
print "too many loops in sample."
# print sample_box_false[:10]
# print sample_box_false.shape[0]
# print sample_iou_false[:10]
print "average iou: ", np.mean(sample_iou_false)
boxes_train.append(sample_box_false)
iou_train.append(sample_iou_false)
try:
Q = 0.2
sample_box_false, sample_iou_false = filter.sample_iou(
pred_box, Q, 0.01, 0.2, new_false / 2, 0, thre_max_neg)
except OverflowError as e:
print "too many loops in sample."
# print sample_box_false[:10]
# print sample_box_false.shape[0]
# print sample_iou_false[:10]
print "average iou: ", np.mean(sample_iou_false)
boxes_train.append(sample_box_false)
iou_train.append(sample_iou_false)
boxes_train = np.vstack(boxes_train)
iou_train = np.vstack(iou_train)
y_train_true = np.ones((new_true, ))
y_train_false = np.zeros((new_false, ))
y_train = np.hstack([y_train_true, y_train_false])
# permutation
ind_perm = np.random.permutation(range(new_false + new_true))
boxes_train = boxes_train[ind_perm, :]
y_train = y_train[ind_perm]
new_y = np.zeros(y_train.shape)
new_y[...] = y_train
ind_pos = np.where(y_train == 1)[0]
ind_neg = np.where(y_train == 0)[0]
vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0])
features = vggnet.get_features_first_raw(frame_data,
boxes_raw=boxes_train,
id=id)
for k, v in features.iteritems():
print k, v.shape
if k == 'f3':
v_pca3 = pca3.transform(v)
#random substitude
pca3_cur_pos = v_pca3[ind_pos, :]
pca3_cur_neg = v_pca3[ind_neg, :]
to_subst = random.sample(range(num_true), new_true)
pca3_pos[to_subst, :] = pca3_cur_pos
to_subst = random.sample(range(num_false), new_false)
pca3_neg[to_subst, :] = pca3_cur_neg
if conf_max < 0.1 and fail_times >= 2: #0.99
#if conf_max<0.95 and conf_max>0.5:
#update classification model
print 'updating model...'
pca3_train = np.vstack([pca3_pos, pca3_neg])
y_train_true = np.ones((num_true, ))
y_train_false = np.zeros((num_false, ))
y_train = np.hstack([y_train_true, y_train_false])
# permutation
ind_perm = np.random.permutation(range(num_false + num_true))
pca3_train = pca3_train[ind_perm, :]
y_train = y_train[ind_perm]
#logistic regression
clf3.fit(pca3_train, y_train)
print 'score is: ', clf3.score(pca3_train, y_train)
#fail_times=0
# (B,G,R)
frame_data_cv = frame_data * 255 # [0,1]-->[0,255]
frame_data_cv = frame_data_cv[:, :, ::-1] # RGB->BGR
frame_data_cv = frame_data_cv.astype('uint8')
#cv2.rectangle(frame_data_cv, (int(gt_box[0]), int(gt_box[1])), (int(gt_box[2]), int(gt_box[3])),
# (255, 0, 0), 2, 1)
if id > 0:
cv2.rectangle(frame_data_cv,
(int(pred_box[0, 0]), int(pred_box[0, 1])),
(int(pred_box[0, 2]), int(pred_box[0, 3])),
(0, 255, 0), 2, 1)
show_particles = False
if show_particles:
for i in range(filter.num_particles):
cx = filter.particles[i, 0]
cy = filter.particles[i, 1]
cv2.circle(frame_data_cv, (int(cx), int(cy)),
1, (0, 0, 255),
thickness=1)
show_box = False
if show_box:
n = 0
for i in ind_pos:
if n % 5 == 0:
cv2.rectangle(
frame_data_cv,
(int(boxes_train[i, 0]), int(boxes_train[i, 1])),
(int(boxes_train[i, 2]), int(boxes_train[i, 3])),
(0, 0, 255), 2, 1)
n += 1
n = 0
show_particles_init = False
if show_particles_init:
for i in range(filter.num_particles):
cx = filter.particles[i, 0]
cy = filter.particles[i, 1]
cv2.circle(frame_data_cv, (int(cx), int(cy)),
1, (0, 255, 0),
thickness=1)
show_frame = False
#cv2.circle(frame_data_cv, (int(filter.cur_c[0]), int(filter.cur_c[1])), 2, (0, 0, 255), thickness=1)
if show_frame:
cv2.imshow(sequence, frame_data_cv)
c = cv2.waitKey(1)
if c != -1:
if chr(c) == 'p':
c = cv2.waitKey()
#print 'You press: ',chr(c)
#if chr(c)=='c':
if chr(c) == 'c':
cv2.destroyWindow(sequence)
#conf_hist=np.array(conf_hist)
#iou_hist=np.array(iou_hist)
#np.save('conf_hist.npy',conf_hist)
#np.save('iou_hist.npy',iou_hist)
break
else:
video_writer.write(frame_data_cv)
end_time = time.time()
video_writer.release()
iou_hist = np.array(iou_hist)
pred_hist = np.array(pred_hist).squeeze()
print "iou_hist: ", iou_hist.shape
print "pred_hist: ", pred_hist.shape
print "get_boxes: ", gt_boxes.shape
precisions, auc_pre = utils.calc_prec(gt_boxes, pred_hist)
print "precision is: %f" % (precisions[19])
suc, auc_iou = utils.calc_success(iou_hist)
print "Average IOU is: %f" % (np.mean(iou_hist))
print "Auc of precision is: %f" % (auc_pre)
print "Auc of success is: %f" % auc_iou
print "Reinit times: %d" % reinit
print "Average FPS: %f" % ((id + 1) / (end_time - start_time))
gt = (gt * 255).long().squeeze_(dim=1)
image = image.to(device)
gt = gt.to(device)
salient_map = salient_map.to(device)
# print(torch.unique(gt), gt.shape)
x = torch.cat((image, salient_map), dim=1)
optimizer.zero_grad()
y, pbr_y = model(x)
# loss = loss_func(y, gt)
loss = loss_func(y, gt) + loss_func(pbr_y, gt)
loss.backward()
optimizer.step()
y_label = softmax_to_label(y)
pbr_y_label = softmax_to_label(pbr_y)
y_iou = calc_iou(y_label, gt)
pbr_y_iou = calc_iou(pbr_y_label, gt)
losses.append(loss.item())
pbr_y_ious.append(pbr_y_iou.item())
y_ious.append(y_iou.item())
print(
f"train: epoch {i+1}, step {index + 1}, loss: {loss}, iou: {y_iou}, pbr iou: {pbr_y_iou}."
)
# break
scheduler.step()
# print(optimizer.state_dict()['param_groups'][0]['lr']) # learning rate
torch.save(model.state_dict(), f"./models/checkpoints.pth")
print(
f"epoch {i+1} training end, mean loss: {np.mean(losses)}, iou: {np.mean(y_ious)}, pbr iou: {np.mean(pbr_y_ious)}."
)
def main(args):
vis = args.vis
debug = args.debug
save = args.save
nparticles = args.particles
root_path = '/home/ccjiang/Documents/py-faster-rcnn/caffe-fast-rcnn/examples/tracker/'
dataset_path = args.dataset # "/data/OTB100"
dataset100_seq = ['Bird2', 'BlurCar1', 'BlurCar3', 'BlurCar4', 'Board', 'Bolt2', 'Boy', 'Car2',
'Car24', 'Coke', 'Coupon', 'Crossing', 'Dancer', 'Dancer2', 'David2', 'David3',
'Dog', 'Dog1', 'Doll', 'FaceOcc1', 'FaceOcc2', 'Fish', 'FleetFace', 'Football1',
'Freeman1', 'Freeman3', 'Girl2', 'Gym', 'Human2', 'Human5', 'Human7', 'Human8',
'Jogging', 'KiteSurf', 'Lemming', 'Man', 'Mhyang', 'MountainBike', 'Rubik',
'Singer1', 'Skater', 'Skater2', 'Subway', 'Suv', 'Tiger1', 'Toy', 'Trans',
'Twinnings', 'Vase']
dataset50_seq = ['Basketball', 'Bird1', 'BlurBody', 'BlurCar2', 'BlurFace', 'BlurOwl',
'Bolt', 'Box', 'Car1', 'Car4', 'CarDark', 'CarScale', 'ClifBar', 'Couple', 'Crowds','David',
'Deer', 'Diving', 'DragonBaby', 'Dudek', 'Football', 'Freeman4', 'Girl',
'Human3', 'Human4', 'Human6', 'Human9', 'Ironman', 'Jump', 'Jumping', 'Liquor',
'Matrix', 'MotorRolling', 'Panda', 'RedTeam', 'Shaking', 'Singer2', 'Skating1',
'Skating2', 'Skiing', 'Soccer', 'Surfer', 'Sylvester', 'Tiger2', 'Trellis',
'Walking', 'Walking2', 'Woman']
datafull_seq = dataset100_seq + dataset50_seq
if "OTB50" in dataset_path:
data_seq = dataset50_seq
else:
data_seq = dataset100_seq
log_name = 'log_1119.txt'
log_file = open(log_name, 'w')
records_success = [] # defaultdict(list)
records_precision = [] # defaultdict(list)
records_reinit = defaultdict(list)
model_def = os.path.join(root_path, args.prototxt)
model_weight = os.path.join(root_path, args.caffemodel)
vggnet = VGGnet.VGGnet(model_def, model_weight)
thre_max_neg = 0.3 # 0.5
test_times = 1 # 0
for t in range(test_times):
print 'Test round: %d' % t
log_file.write('Test round: %d\n' % t)
# sequences = ['Fish']
for sequence in datafull_seq: # datafull_seq
if sequence in dataset50_seq:
dataset_path = "/data/OTB50"
else:
dataset_path = "/data/OTB100"
for t in os.walk(os.path.join(dataset_path, sequence, sequence, "img")):
if t[0] == os.path.join(dataset_path, sequence, sequence, "img"):
nFrame = len(t[2])
print 'Processing: %s' % sequence
log_file.write('Processing: %s\n' % sequence)
print "Total frames are: ", nFrame
log_file.write('Total frames are: %d\n' % nFrame)
gt_path = os.path.join(dataset_path, sequence, sequence, "groundtruth_rect.txt")
gt_boxes = utils.get_boxes_all(gt_path)
conf_hist = []
iou_hist = []
area_hist = []
pred_hist = [] # (x1,y1,x2,y2)
eig_hist = []
reinit = 0
nFrame = np.minimum(nFrame, gt_boxes.shape[0])
id_shift = 0
init_id = False
update_recent = False
for id in np.arange(0, nFrame):
frame_name = "img/%04d.jpg" % (id + 1)
# print "Start processing: %s" % frame_name
frame_path = os.path.join(dataset_path, sequence, sequence, frame_name)
if os.path.exists(frame_path) == False:
id_shift = id_shift + 1
continue
id = id - id_shift
frame_data = caffe.io.load_image(frame_path) # (432,576,3), in [0,1]
gt_box = gt_boxes[id]
if init_id == False:
h, w, c = frame_data.shape
frame_shape = [c, w, h]
fps = 20
fourcc = cv2.VideoWriter_fourcc(*'MJPG')
video_writer = cv2.VideoWriter("res_%s.avi"%sequence,fourcc,fps,(w,h))
fail_times = 0
box_w = gt_box[2] - gt_box[0]
box_h = gt_box[3] - gt_box[1]
area = (gt_box[2] - gt_box[0]) * (gt_box[3] - gt_box[1])
ratio = (gt_box[2] - gt_box[0]) / (gt_box[3] - gt_box[1]) # ratio=w/h
# set up net.blobs['im_info']
print "Image Size: ", w, h
log_file.write('Image Size: %d %d\n' % (w, h))
b = gt_box[np.newaxis, :]
vggnet.reshape(w=w, h=h, nbox=b.shape[0])
features0 = vggnet.get_features("conv3_3", frame_data,
boxes_raw=b) # shape:(256,hs,ws),conv3_3,res3b3
features0 = np.squeeze(features0)
pca_f, scaler_f = featmap_pca2(features0,ncompnents=128)#128
box_w = gt_box[2] - gt_box[0]
box_h = gt_box[3] - gt_box[1]
vggnet.reshape(w=w, h=h, nbox=nparticles)
pfilter = PFfilter.PFfilter(utils.bbox_to_states(gt_box, area, ratio), area, ratio, w, h,
nparticles)
pfilter.create_particles()
pfilter.restrict_particles(w, h)
area_hist.append(pfilter.cur_a)
pred_hist.append(np.array(gt_box).reshape(1, -1))
# pca
# test sample_iou
num_true = 500
num_false = 1000 # 1000
#data augument
gt_box_otb = gt_box.copy()
gt_box_otb[2] -= gt_box_otb[0]
gt_box_otb[3] -= gt_box_otb[1]
boxes_train = []
ids = np.zeros(num_false + num_true)
imgs = []
for i in np.arange(4):
if i == 0:
img1,gt1,img2,gt2,img3,gt3 = DataAugment(frame_data,gt_box_otb,True)
gt1[2] += gt1[0]
gt1[3] += gt1[1]
gt2[2] += gt2[0]
gt2[3] += gt2[1]
gt3[2] += gt3[0]
gt3[3] += gt3[1]
box_true1, iou_true = pfilter.sample_iou_pred_box(gt1, 0.05, 0.01, 0.01, 20, 0.8, 1.0)
box_true2, iou_true = pfilter.sample_iou_pred_box(gt2, 0.05, 0.01, 0.01, 40, 0.8, 1.0)
box_true3, iou_true = pfilter.sample_iou_pred_box(gt3, 0.05, 0.01, 0.01, 20, 0.8, 1.0)
box_true1[0, ...] = gt1
box_true2[0, ...] = gt2
box_true3[0, ...] = gt3
boxes_train.append(box_true1)
boxes_train.append(box_true2)
boxes_train.append(box_true3)
imgs.append(img1)
imgs.append(img2)
imgs.append(img3)
ids[20:60] = 1
ids[60:80] = 2
else:
img1, gt1, img2, gt2 = DataAugment(frame_data, gt_box_otb,False)
gt1[2] += gt1[0]
gt1[3] += gt1[1]
gt2[2] += gt2[0]
gt2[3] += gt2[1]
box_true1, iou_true = pfilter.sample_iou_pred_box(gt1, 0.05, 0.01, 0.01, 20, 0.8, 1.0)
box_true2, iou_true = pfilter.sample_iou_pred_box(gt2, 0.05, 0.01, 0.01, 20, 0.8, 1.0)
box_true1[0, ...] = gt1
box_true2[0, ...] = gt2
boxes_train.append(box_true1)
boxes_train.append(box_true2)
imgs.append(img1)
imgs.append(img2)
cur_i = 80+(i-1)*40
ids[cur_i:(cur_i+20)] = 3+(i-1)*2
ids[(cur_i+20):(cur_i+40)] = 3+(i-1)*2+1
# boxes_train_neg=[]
try:
# Q=[[1,0],[0,1]] #for pixel wise
Q = 0.05 # box_w,box_h,0.05
sample_box_true, sample_iou_true = pfilter.sample_iou_pred_box(gt_box, Q, 0.01, 0.01,
num_true-200, 0.8,
1.0)#0.8
except OverflowError as e:
print "too many loops in sample in Initialize--TRUE."
boxes_train.append(sample_box_true)
try:
# Q=[[36,0],[0,36]]#for pixel wise
Q = 0.2 # 0.2
sample_box_false, sample_iou_false = pfilter.sample_iou(gt_box, Q, 0.2, 0.01, num_false / 2, 0,
thre_max_neg) # 0.2,0.01
except OverflowError as e:
print "too many loops in sample in Initialize--FALSE."
# print sample_box_false[:10]
# print sample_box_false.shape[0]
# print sample_iou_false[:10]
# print "average iou: ", np.mean(sample_iou_false)
boxes_train.append(sample_box_false)
try:
# Q=[[36,0],[0,36]]#for pixel wise
Q = 0.2 # 0.2
sample_box_false, sample_iou_false = pfilter.sample_iou(gt_box, Q, 0.01, 0.2, num_false / 2, 0,
thre_max_neg) # 0.01,0.2
except OverflowError as e:
print "too many loops in sample in Initialize--FALSE."
boxes_train.append(sample_box_false)
boxes_train = np.vstack(boxes_train)
imgs.append(frame_data)
imgs = np.stack(imgs,axis=0)#(10,h,w,c)
ids[200:] = 9
y_train_true = np.ones((num_true,))
y_train_false = np.zeros((num_false,))
ids_save = np.ones((num_true+num_false))
ids_save[num_true:] = 0
ids_save[20:60] = 2
y_train = np.hstack([y_train_true, y_train_false])
# permutation
ind_perm = np.random.permutation(range(num_false + num_true))
boxes_train = boxes_train[ind_perm, :]
ids_save = ids_save[ind_perm]
y_train = y_train[ind_perm]
ids = ids[ind_perm]
ind_pos = np.where(y_train == 1)[0]
ind_neg = np.where(y_train == 0)[0]
vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0],batch_size=10)
#features = vggnet.get_features_first_raw(frame_data, boxes_raw=boxes_train, id=id)
features = vggnet.get_features_first_id(imgs,boxes_raw=boxes_train,id=ids)
#features = vggnet.get_features_first_sel(frame_data, boxes_raw=boxes_train, id=id, sel=f_inds)
for k, v in features.iteritems():
# print k,v.shape
if k == 'f3':
#pca3, scaler1, nPCA = utils.skl_pca2(v)
v = feat_transformpca(pca_f, scaler_f, v) # (N,128,7,7)
#pca3,scaler,nPCA=utils.skl_pca2(v)
#v_pca3 = pca3.transform(scaler.transform(v))
#np.save("pca_results/testpca_%s.npy"%sequence,v_pca3)
#np.save('labelpca.npy',y_train)
#np.save("pca_results/label_pca_%s"%sequence,ids_save)
pca3_pos = np.zeros((num_true, pca_f.n_components_*49), dtype=np.float32)
pca3_neg = np.zeros((num_false, pca_f.n_components_*49), dtype=np.float32)
pca3_pos[...] = v[ind_pos, :]
pca3_neg[...] = v[ind_neg, :]
# utils.vis_as_image(v_pca3)
# plt.imshow(v_pca3)
# plt.title("PCA features")
# plt.show()
# plt.close()
# logistic regression
y_weight = sklearn.utils.class_weight.compute_class_weight(class_weight='balanced',
classes=np.array([0, 1]),
y=y_train)
# print y_weight
class_weight = {0: y_weight[0], 1: y_weight[1]}
clf3 = linear_model.LogisticRegression(fit_intercept=True, solver='liblinear')
clf3.fit(v, y_train)
vis_feature = False
if vis_feature:
utils.vis_features(features, id)
start_time = time.time()
else:
if fail_times >= 5:
# reinitialize
update_recent = False
reinit += 1
area = (gt_box[2] - gt_box[0]) * (gt_box[3] - gt_box[1])
ratio = (gt_box[2] - gt_box[0]) / (gt_box[3] - gt_box[1])
pfilter = PFfilter.PFfilter(utils.bbox_to_states(gt_box, area, ratio), area, ratio, w, h,
nparticles)
# filter.reset(utils.bbox_to_states(gt_box, area, ratio), area, ratio)
pfilter.create_particles()
pfilter.restrict_particles(w, h)
area_hist.append(pfilter.cur_a)
pred_box = gt_box
pred_hist.append(np.array(pred_box).reshape(1, -1))
conf_hist.append(-0.1)
boxes_train = []
# boxes_train_neg=[]
iou_train = []
try:
# Q=[[1,0],[0,1]] #for pixel wise
Q = 0.05 # box_w,box_h,0.05
sample_box_true, sample_iou_true = pfilter.sample_iou_pred_box(gt_box, Q, 0.01, 0.01,
num_true, 0.8,
1.0)
except OverflowError as e:
print "too many loops in sample in Reinitialize--TRUE."
boxes_train.append(sample_box_true)
iou_train.append(sample_iou_true)
try:
# Q=[[36,0],[0,36]]#for pixel wise
Q = 0.2 # 0.2
sample_box_false, sample_iou_false = pfilter.sample_iou(gt_box, Q, 0.01, 0.2, num_false / 2,
0,
thre_max_neg)
except OverflowError as e:
print "too many loops in sample in Reinitialize--FALSE."
boxes_train.append(sample_box_false)
iou_train.append(sample_iou_false)
try:
# Q=[[36,0],[0,36]]#for pixel wise
Q = 0.2 # 0.2
sample_box_false, sample_iou_false = pfilter.sample_iou(gt_box, Q, 0.2, 0.01, num_false / 2,
0,
thre_max_neg)
except OverflowError as e:
print "too many loops in sample in Reinitialize--FALSE."
# print sample_box_false[:10]
# print sample_box_false.shape[0]
# print sample_iou_false[:10]
# print "average iou: ", np.mean(sample_iou_false)
boxes_train.append(sample_box_false)
iou_train.append(sample_iou_false)
boxes_train = np.vstack(boxes_train)
iou_train = np.vstack(iou_train)
y_train_true = np.ones((num_true,))
y_train_false = np.zeros((num_false,))
y_train = np.hstack([y_train_true, y_train_false])
# permutation
ind_perm = np.random.permutation(range(num_false + num_true))
boxes_train = boxes_train[ind_perm, :]
iou_train = iou_train[ind_perm]
y_train = y_train[ind_perm]
ind_pos = np.where(y_train == 1)[0]
ind_neg = np.where(y_train == 0)[0]
vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0])
features = vggnet.get_features_first_raw(frame_data, boxes_raw=boxes_train, id=id)
#features=feat_transformpca(pca_f,scaler_f,features)
#features = vggnet.get_features_first_sel(frame_data, boxes_raw=boxes_train, id=id, sel=f_inds)
for k, v in features.iteritems():
# print k, v.shape
if k == 'f3':
v = feat_transformpca(pca_f, scaler_f, v) # (N,128,7,7)
#v_pca3 = pca3.transform(scaler.transform(v))
pca3_pos[...] = v[ind_pos, :]
pca3_neg[...] = v[ind_neg, :]
clf3 = linear_model.LogisticRegression(fit_intercept=True, solver='liblinear')
clf3.fit(v, y_train)
# score3 = clf3.score(v_pca3, y_train)
# print 'score3: ', score3
# prob=clf3.predict_proba(v_pca3)
# print clf3.classes_
fail_times = 0
continue
pfilter.predict_particles(Q=0.2, cr=0.01, ca=0.01) # 0.2,0.01
pfilter.restrict_particles(w, h)
area_hist.append(pfilter.cur_a)
# compute conf
# conf = np.zeros(pfilter.weights.shape)
# np.save('particles.npy',filter.particles)
pred_boxes = utils.state_to_bbox(pfilter.particles, area, ratio)
#add Gaussian regularization
if id>1:
gauss_sig = 0.5
gauss_w = np.exp(-np.square((pfilter.particles[:,0]-pred_state[0])/(gauss_sig*box_w)/2.0)-np.square((pfilter.particles[:,1]-pred_state[1])/(gauss_sig*box_h)/2.0))
pfilter.update_particles(gauss_w)
print gauss_w
vggnet.reshape(w, h, pfilter.num_particles)
features = vggnet.get_features_first_raw(frame_data, boxes_raw=pred_boxes, id=id)
#features=feat_transformpca(pca_f,scaler_f,features)
#features = vggnet.get_features_first_sel(frame_data, boxes_raw=pred_boxes, id=id, sel=f_inds)
for k, v in features.iteritems():
# print k,v.shape
if k == 'f3':
v = feat_transformpca(pca_f, scaler_f, v) # (N,128,7,7)
vf = v
#v_pca3 = pca3.transform(scaler.transform(v))
conf = clf3.predict_proba(v)[:, 1]
# process preds to find out pred_box in terms of confm
conf = np.array(conf)
conf_max = np.max(conf)
conf_min = np.min(conf)
pfilter.update_particles(conf)
# do resample first or estimate first?
# filter.resample() # always resample
pred_state, s_particles, r_particles = pfilter.estimate(k=10)
pfilter.resample()
pred_box = utils.state_to_bbox(pred_state.reshape((-1, 6)), area, ratio)
hard,hard_negv = nms_pred(pred_box,pred_boxes,vf,conf)
if hard:
hard_negvN = hard_negv.shape[0]
#print hard_negv.shape
else:
hard_negvN = 0
avg_pos = np.mean(pfilter.particles[:, :2], axis=0, keepdims=True)
# avg_pos[:,0]/=w
# avg_pos[:,1]/=h
ptls_avg = (pfilter.particles[:, :2] - avg_pos) / np.array([[box_w, box_h]])
cov_particles = np.dot(ptls_avg.T, ptls_avg) / pfilter.particles.shape[
0]
eigval, eigvec = np.linalg.eig(cov_particles)
max_val = eigval[0]
eig_hist.append(max_val)
print 'Max eigvalue: %f' % max_val
# print 'conf is: ',conf
if conf_max > 0.5: # 0.8
fail_times = 0
update_recent = False
else:
fail_times += 1
show_sr = False
if show_sr:
count, xedge, yedge, tmp_im = plt.hist2d(s_particles, r_particles, bins=10,
weights=pfilter.weights.squeeze(), cmap=plt.cm.gray)
top3 = np.argsort(-count, axis=None)[:3]
row_ind = top3[:] / count.shape[1]
col_ind = top3[:] % count.shape[0]
plt.show()
print pred_box
iou = utils.calc_iou(gt_box, pred_box)
# print 'iou is: ', iou
pred_hist.append(pred_box)
conf_hist.append(conf_max)
iou_hist.append(iou)
if conf_max >= 0.7: # 0.5
# update pca3_pos and pca3_neg
new_true = 100 # 100
new_false = 400 # 200
boxes_train = []
iou_train = []
Q = 0.05 # 0.02
try:
sample_box_true, sample_iou_true = pfilter.sample_iou_pred_box(pred_box, Q, 0.01, 0.01,
new_true,
0.85,
1.0)
except OverflowError as e:
print "too many loops in sample in Update--TRUE."
# print sample_box_true[:10]
# print sample_box_true.shape[0]
# print sample_iou_true[:10]
# print "average iou: ", np.mean(sample_iou_true)
boxes_train.append(sample_box_true)
iou_train.append(sample_iou_true)
# part_iou=utils.calc_iou(pred_box,pred_boxes)
# ind_iou=np.where(part_iou<0.3)[0]
# ind_n=np.minimum(new_false/2,ind_iou.shape[0])
# boxes_train.append(pred_boxes[ind_iou[:ind_n],:])
# iou_train.append(part_iou[ind_iou])
new_false_left = new_false - hard_negvN # -ind_n
try:
Q = 0.2 # 0.2
sample_box_false, sample_iou_false = pfilter.sample_iou_pred_box(pred_box, Q, 0.2, 0.01,
(new_false_left + 1) / 2,
0, thre_max_neg)
except OverflowError as e:
print "too many loops in sample in Update--FALSE."
# print sample_box_false[:10]
# print sample_box_false.shape[0]
# print sample_iou_false[:10]
# print "average iou: ", np.mean(sample_iou_false)
boxes_train.append(sample_box_false)
iou_train.append(sample_iou_false)
try:
Q = 0.2 # 0.2
sample_box_false, sample_iou_false = pfilter.sample_iou_pred_box(pred_box, Q, 0.01, 0.2,
new_false_left / 2, 0,
thre_max_neg)
except OverflowError as e:
print "too many loops in sample in Update--FALSE."
boxes_train.append(sample_box_false)
iou_train.append(sample_iou_false)
boxes_train = np.vstack(boxes_train)
# iou_train = np.vstack(iou_train)
vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0])
features = vggnet.get_features_second_raw(boxes_raw=boxes_train, id=id)
#features = feat_transformpca(pca_f,scaler_f,features)
#features = vggnet.get_features_second_sel(boxes_raw=boxes_train, id=id, sel=f_inds)
for k, v in features.iteritems():
# print k, v.shape
if k == 'f3':
v = feat_transformpca(pca_f, scaler_f, v) # (N,128,7,7)
#v_pca3 = pca3.transform(scaler.transform(v))
if hard:
print v.shape
print hard_negv.shape
v = np.vstack([v,hard_negv])
y_train_true = np.ones((new_true,))
y_train_false = np.zeros((new_false,))
y_train = np.hstack([y_train_true, y_train_false])
# permutation
ind_perm = np.random.permutation(range(new_false + new_true))
#boxes_train = boxes_train[ind_perm, :]
v = v[ind_perm,:]
y_train = y_train[ind_perm]
new_y = np.zeros(y_train.shape)
new_y[...] = y_train
ind_pos = np.where(y_train == 1)[0]
ind_neg = np.where(y_train == 0)[0]
# random substitude
pca3_cur_pos = v[ind_pos, :]
pca3_cur_neg = v[ind_neg, :]
to_subst = random.sample(range(num_true), new_true)
pca3_pos[to_subst, :] = pca3_cur_pos
to_subst = random.sample(range(num_false), new_false)
pca3_neg[to_subst, :] = pca3_cur_neg
if conf_max < 1 and fail_times >= 2 and update_recent==False:
# if id%10==0:
update_recent = True
pca3_train = np.vstack([pca3_pos, pca3_neg])
y_train_true = np.ones((num_true,))
y_train_false = np.zeros((num_false,))
y_train = np.hstack([y_train_true, y_train_false])
# permutation
ind_perm = np.random.permutation(range(num_false + num_true))
pca3_train = pca3_train[ind_perm, :]
y_train = y_train[ind_perm]
# logistic regression
clf3 = linear_model.LogisticRegression(fit_intercept=True, solver='liblinear')
clf3.fit(pca3_train, y_train)
# print 'score is: ',clf3.score(pca3_train,y_train)
# (B,G,R)
frame_data_cv = frame_data * 255 # [0,1]-->[0,255]
frame_data_cv = frame_data_cv[:, :, ::-1] # RGB->BGR
frame_data_cv = frame_data_cv.astype('uint8')
cv2.rectangle(frame_data_cv, (int(gt_box[0]), int(gt_box[1])), (int(gt_box[2]), int(gt_box[3])),
(255, 0, 0), 2, 1)
if id > 0 and init_id == True:
cv2.rectangle(frame_data_cv, (int(pred_box[0, 0]), int(pred_box[0, 1])),
(int(pred_box[0, 2]), int(pred_box[0, 3])),
(0, 255, 0), 2, 1)
if init_id == False:
init_id = True
show_particles = False
if show_particles:
for i in range(filter.num_particles):
cx = pfilter.particles[i, 0]
cy = pfilter.particles[i, 1]
cv2.circle(frame_data_cv, (int(cx), int(cy)), 1, (0, 0, 255), thickness=1)
show_box = False
if show_box:
n = 0
for i in ind_pos:
if n % 5 == 0:
cv2.rectangle(frame_data_cv, (int(boxes_train[i, 0]), int(boxes_train[i, 1])),
(int(boxes_train[i, 2]), int(boxes_train[i, 3])), (0, 0, 255), 2, 1)
n += 1
n = 0
show_particles_init = False
if show_particles_init:
for i in range(filter.num_particles):
cx = pfilter.particles[i, 0]
cy = pfilter.particles[i, 1]
cv2.circle(frame_data_cv, (int(cx), int(cy)), 1, (0, 255, 0), thickness=1)
show_frame = False
cv2.circle(frame_data_cv, (int(pfilter.cur_c[0]), int(pfilter.cur_c[1])), 2, (0, 0, 255), thickness=1)
if show_frame:
cv2.imshow(sequence, frame_data_cv)
c = cv2.waitKey(1)
if c != -1:
cv2.destroyWindow(sequence)
break
else:
video_writer.write(frame_data_cv)
end_time = time.time()
video_writer.release()
print "Average FPS: %f" % (nFrame / (end_time - start_time))
log_file.write("Average FPS: %f\n" % (nFrame / (end_time - start_time)))
conf_hist = np.array(conf_hist)
iou_hist = np.array(iou_hist)
area_hist = np.array(area_hist)
pred_hist = np.vstack(pred_hist)
precisions, auc_pre = utils.calc_prec(gt_boxes, pred_hist)
suc, auc_iou = utils.calc_success(iou_hist)
records_precision.append(precisions * nFrame)
records_success.append(suc * nFrame)
print 'Precision @20 is: %f' % precisions[19]
print 'Auc of Precision is: %f' % auc_pre
print 'Auc of Success is: %f' % auc_iou
print 'Reinit times: %d' % reinit
log_file.write("Precision @20 is: %f\n" % precisions[19])
log_file.write('Auc of Precision is: %f\n' % auc_pre)
log_file.write('Auc of Success is: %f\n' % auc_iou)
log_file.write('Reinit times: %d\n' % reinit)
#log_file.write('Selected feature maps: %d\n' % f_inds.shape[0])
#log_file.write('PCA components: %d\n' % nPCA)
#res_f = open('results11/%s.txt'%sequence,'w')
#pred_hist[:,2:] = pred_hist[:,2:] - pred_hist[:,:2]
#res_f = write_res(pred_hist,res_f)
#res_f.close()
log_file.close()
pkl = open('results_1031.pkl', 'w')
pickle.dump([records_precision, records_success], pkl)
pkl.close()
def _evaluate_worker(image_path, cfg, weights, output_names, input_size_wh,
score_thresh, iou_thresh, labels, save_size_wh,
save_dir_path):
"""
Need to add False Positive
"""
image_path = image_path.strip()
image = cv2.imread(image_path)
net = cv2.dnn.readNetFromDarknet(cfg, weights)
idxs, boxes, scores, class_ids = \
infer.infer_image(image, net, output_names, input_size_wh, score_thresh)
label_filename = os.path.splitext(image_path)[0] + '.txt'
all_test_pass = True
f_label = list(open(label_filename, 'r'))
for label in f_label:
label = label.strip().split(' ')
t_class_id = int(label[0])
t_cbox = np.array(tuple(map(float, label[1:5])))
t_cbox = t_cbox * np.array([*input_size_wh, *input_size_wh])
t_bbox = utils.cbox2bbox(t_cbox)
label_test_pass = False
if len(idxs) >= 1:
for i in idxs.flatten():
if t_class_id != class_ids[i]:
continue
cbox = boxes[i][:4] * np.array([*input_size_wh, *input_size_wh])
bbox = utils.cbox2bbox(cbox)
iou = utils.calc_iou(bbox, t_bbox)
if iou >= iou_thresh:
label_test_pass = True
break
if not label_test_pass:
all_test_pass = False
break
if not all_test_pass:
save_image = cv2.resize(image, save_size_wh)
save_image_true = save_image.copy()
save_image_pred = save_image.copy()
for label in f_label:
label = label.strip().split(' ')
t_class_id = int(label[0])
t_cbox = np.array(tuple(map(float, label[1:5])))
t_cbox = t_cbox * np.array([*save_size_wh, *save_size_wh])
t_x1, t_y1, t_x2, t_y2 = list(map(int, utils.cbox2bbox(t_cbox)))
cv2.rectangle(
save_image_true, (t_x1,t_y1), (t_x2,t_y2), (0,255,0), 2)
text = f'{labels[t_class_id]}'
cv2.putText(
save_image_true, text, (t_x1+5, t_y2-15),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0,255,0), 2)
if len(idxs) >= 1:
for i in idxs.flatten():
cbox = boxes[i][:4] * np.array([*save_size_wh, *save_size_wh])
x1, y1, x2, y2 = list(map(int, utils.cbox2bbox(cbox)))
cv2.rectangle(save_image_pred, (x1,y1), (x2,y2), (0,255,0), 2)
label = f'{labels[class_ids[i]]}'
cv2.putText(
save_image_pred, label, (x1+5, y2-15),
cv2.FONT_HERSHEY_SIMPLEX, 1, (0,255,0), 2)
score = f'{scores[i]:.2f}'
cv2.putText(
save_image_pred, score, (x1+5, y2+15),
cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,255,0), 1)
stack_image = np.vstack((save_image, save_image_true, save_image_pred))
cv2.imwrite(
os.path.join(save_dir_path, os.path.basename(image_path)),
stack_image)
return 1
return 0
def train():
torch.manual_seed(1280)
torch.cuda.manual_seed(1280)
np.random.seed(1280)
# setup device
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print("device = {}".format(device))
# load data
data_loader_train = Cityscapes_Loader(path_root=path_root,split="train",n_classes=n_class)
data_loader_val = Cityscapes_Loader(path_root=path_root,split="val",n_classes=n_class)
train_loader = data.DataLoader(data_loader_train,batch_size=batch_size,num_workers=16)
val_loader = data.DataLoader(data_loader_val,batch_size=batch_size,num_workers=16) # val batch_size=1
# set model
assert torch.cuda.is_available(), "先把下面的cuda()删掉，debug阶段，不支持cpu"
# if torch.cuda.is_available():
# torch.backends.cudnn.benchmark = True
pretrain_model = VGGNet(requires_grad=True, remove_fc=True)
model = FCNs(pretrained_net=pretrain_model, n_class=n_class)
print("model loading success...")
# set model running devices
model.to(device)
model = torch.nn.DataParallel(model, device_ids=range(torch.cuda.device_count()))
print("usable gpu num: {}".format(torch.cuda.device_count()))
# set optimizer, lr_scheduler, loss function
optimizer = None
if optimizer_name == "rmsprop":
optimizer = torch.optim.RMSprop(model.parameters(),lr=learn_rate,momentum=momentum,weight_decay=weight_decay)
elif optimizer_name == "sgd":
optimizer = torch.optim.SGD(model.parameters(),lr=learn_rate,momentum=momentum,weight_decay=weight_decay)
elif optimizer_name == "adam":
optimizer = torch.optim.Adam(model.parameters(),lr=learn_rate)
#criterion = mybasilLoss()
criterion = torch.nn.BCEWithLogitsLoss()
lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=step_size, gamma=gamma)
# load checkpoints
last_best_iou = -100.
load_ckpt_path = os.path.join("checkpoints",load_ckpt_name)
if is_load_checkpoints:
if torch.cuda.is_available():
checkpoint = torch.load(load_ckpt_path)
else:
checkpoint = torch.load(load_ckpt_path, map_location='cpu')
model.load_state_dict(checkpoint['model_state'])
last_best_iou = checkpoint['best_iou']
start_epoch = checkpoint['epoch']
print('Checkpoint resume success... last iou:{:.4%}'.format(last_best_iou))
# train epoch
best_iou = last_best_iou
time_epoch = time.time()
i = 0
for epoch in range(epochs):
time_step = time.time()
for step, batch in enumerate(train_loader):
lr_scheduler.step(epoch=epoch) # 这个scheduler放在哪里？上一个for，还是这个for
model.train()
images = batch[0].to(device)
#labels = batch[1].to(device)
targets = batch[2].to(device) # targets.shape=[batch, n_classes, H, W]
optimizer.zero_grad()
outputs = model(images)
loss = None
try:
loss = criterion(input=outputs,target=targets)
except:
torch.cuda.empty_cache()
loss = criterion(input=outputs, target=targets)
loss.backward()
optimizer.step()
if(step%100 == 0):
print("train after setp:{}, Loss:{:.4%}, time used:{:.4} s".format(step,loss.item(),time.time()-time_step))
time_step = time.time()
writer.add_scalars('ForTestOnly_record/Loss', {'train_loss': loss}, i + 1)
i += 1
# each epoch save the checkpoint
model.eval()
with torch.no_grad():
total_iou = 0
for step_val, batch_val in enumerate(val_loader):
images_val = batch_val[0].to(device)
labels_val = batch_val[1].to(device)
targets_val = batch_val[2].to(device)
outputs_val = model(images_val)
loss_val = criterion(input=outputs_val,target=targets_val)
pred = outputs_val.data.max(1)[1].cpu().numpy() # 将mask格式[
gt = labels_val.data.cpu().numpy()
#break # for only one val batch
total_iou += calc_iou(pred,gt,n_class)
mean_iou = total_iou/step_val
print("epoch:{},loss_val:{:.4%},iou:{:.2%},total time used:{:.4}s".format(epoch + 1, loss_val, mean_iou,time.time()-time_epoch))
writer.add_scalars('Train_record/Loss',{'train_loss':loss,'val_loss':loss_val}, epoch+1)
writer.add_scalars('Train_record/iou',{"iou":mean_iou}, epoch+1)
time_epoch = time.time()
if mean_iou >= best_iou:
best_iou = mean_iou
state ={
"epoch":epoch + 1,
"model_state":model.state_dict(),
"best_iou":best_iou,
}
if not os.path.isdir('./checkpoints'):
os.mkdir('./checkpoints')
save_path = os.path.join('./checkpoints', "eph_{}_iou_{:.2%}.ckpt.pth".format(epoch+1,best_iou))
torch.save(state,save_path)
print("checkpoint saved success")
writer.close()
def build_loss_and_optimizer(self):
#
# Classification Loss - SparseSoftmaxCrossEntropyLoss, WeightedSigmoidClassificationLoss
#
self.cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=self.logits, labels=self.y)
self.cross_entropy_loss = tf.reduce_mean(self.cross_entropy)
self.cls_loss = self.cross_entropy_loss
self.pred_class = tf.argmax(self.logits, axis=1)
self.corrections = tf.equal(self.y, self.pred_class)
self.cls_accuracy = tf.reduce_sum(tf.cast(
self.corrections, tf.int32)) / tf.shape(self.y)[0]
# self.accuracy = tf.metrics.accuracy(labels=self.y,
# predictions=tf.argmax(self.logits, axis=1))[1]
#
# Localisation Loss - WeightedSmoothL1LocalizationLoss
#
# top-1 loc,
pred_class = tf.argmax(self.logits, axis=1)
self.cam_t1 = self.get_classmap(pred_class, self.conv)
self.pred_bbox, self.debug = utils.get_bbox_from_cam(self.cam_t1, 0.2)
self.huber_loss_t1 = tf.losses.huber_loss(
self.bbox,
self.pred_bbox) # fixme: pred_cls 가 gt_cls 가 아닌 경우 loss 계산 다르게 해야함
self.loc_loss_t1 = tf.reduce_mean(self.huber_loss_t1)
self.iou_t1 = utils.calc_iou(self.pred_bbox, self.bbox)
t1_iou_over_50 = self.iou_t1 > 0.5
self.correct_and_iou_t1 = tf.logical_and(self.corrections,
t1_iou_over_50)
self.top1_loc_accuracy = tf.reduce_sum(
tf.cast(self.correct_and_iou_t1, tf.float32)) / self.config.bs
# gt-known loc, use this!
self.cam_gt = self.get_classmap(self.y, self.conv)
self.pred_bbox_gt, self.debug_gt = utils.get_bbox_from_cam(
self.cam_gt, 0.2)
self.huber_loss_gt = tf.losses.huber_loss(self.bbox, self.pred_bbox_gt)
self.loc_loss_gt = tf.reduce_mean(self.huber_loss_gt)
self.iou_gt = utils.calc_iou(self.pred_bbox_gt, self.bbox)
gt_iou_over_50 = self.iou_gt > 0.5
self.correct_and_iou_gt = gt_iou_over_50
self.gt_known_loc_accuracy = tf.reduce_sum(
tf.cast(self.correct_and_iou_gt, tf.float32)) / self.config.bs
# Total Loss -- loc_loss_gt
alpha = self.config.alpha
beta = self.config.beta
gamma = self.config.gamma
# self.tot_loss = alpha * self.cls_loss + beta * self.loc_loss_t1
self.tot_loss = alpha * self.cls_loss + beta * self.loc_loss_gt + gamma * self.loc_loss_t1
# Optimizer
if self.config.opt == 'adam':
self.optimizer = tf.train.AdamOptimizer(self.learning_rate,
beta1=self.config.beta1)
elif self.config.opt == 'rmsprop':
self.optimizer = tf.train.RMSPropOptimizer(self.learning_rate)
# grad-clipping
var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope=self.model_name)
gvs = self.optimizer.compute_gradients(self.tot_loss,
var_list=var_list)
capped_gvs = [(tf.clip_by_value(grad, -5., 5.), var)
for grad, var in gvs]
self.train_op = self.optimizer.apply_gradients(capped_gvs)
cls_gvs = self.optimizer.compute_gradients(self.cls_loss,
var_list=var_list)
cls_capped_gvs = [(tf.clip_by_value(grad, -5., 5.), var)
for grad, var in cls_gvs]
self.cls_train_op = self.optimizer.apply_gradients(cls_capped_gvs)
def train(run, epochs):
run_file = open(os.path.join(opt.outputFolder, f"run_{run}.csv"), "w")
epoch_loss = []
val_loss = []
acc = []
miou = []
train_time = 0
ce_loss = nn.CrossEntropyLoss()
optim = torch.optim.Adam(model.parameters(), lr=opt.lr)
start_time = time.time()
for i in range(epochs):
ep = 0
model.train()
for X, y in train_loader:
optim.zero_grad()
X = X.to(device)
y = y.to(device)
output = model(X)['out']
loss = ce_loss(output, y)
loss.backward()
ep += loss.item()
optim.step()
epoch_loss.append(ep)
correct = 0
total = 0
val = 0
iou = []
model.eval()
for X, y in eval_loader:
X = X.to(device)
y = y.to(device)
output = model(X)['out']
loss = ce_loss(output, y)
val += loss.item()
probs = torch.functional.F.softmax(output, 1)
y_pred = torch.argmax(probs, dim=1)
correct += torch.sum(y_pred == y).item()
total += (y.shape[-1] * y.shape[-2])
iou.append(calc_iou(y_pred, y))
val_loss.append(val)
miou.append(np.mean(iou))
acc.append(correct / total)
print(i, "mIoU: ", round(miou[-1], 2), " - Accuracy: ",
round(acc[-1], 2), " - Loss: ", round(val, 1))
run_file.write(f'{i},{round(miou[-1], 2)},{round(val,2)}\n')
# early stopping
if must_stop(miou):
print('***** Early stopping! :)')
break
train_time += time.time() - start_time
print(f"Train_time={train_time/60} (min)")
plt.subplots(figsize=(15, 8))
plt.imsave(os.path.join(opt.outputFolder, f"predicition_{run}.png"),
np.hstack((y_pred.squeeze().cpu(), y.squeeze().cpu())))
save_file(f'val_loss_{run}.txt', val_loss)
save_file(f'epoch_loss_{run}.txt', epoch_loss)
save_file(f'acurracy_{run}.txt', acc)
save_file(f'miou_{run}.txt', miou)
run_file.close()
torch.save(model.state_dict(),
os.path.join(opt.outputFolder, f'model_{run}.pth'))
return val_loss[-1], epoch_loss[-1], acc[-1], miou[-1]
def main(args):
vis = args.vis
debug = args.debug
save = args.save
nparticles = args.particles
root_path = '/home/ccjiang/Documents/caffe-fast-rcnn/examples/tracker/'
dataset_path = args.dataset #"/data/OTB100"
dataset100_seq = [
'Bird2', 'BlurCar1', 'BlurCar3', 'BlurCar4', 'Board', 'Bolt2', 'Boy',
'Car2', 'Car24', 'Coke', 'Coupon', 'Crossing', 'Dancer', 'Dancer2',
'David2', 'David3', 'Dog', 'Dog1', 'Doll', 'FaceOcc1', 'FaceOcc2',
'Fish', 'FleetFace', 'Football1', 'Freeman1', 'Freeman3', 'Girl2',
'Gym', 'Human2', 'Human5', 'Human7', 'Human8', 'Jogging', 'KiteSurf',
'Lemming', 'Man', 'Mhyang', 'MountainBike', 'Rubik', 'Singer1',
'Skater', 'Skater2', 'Subway', 'Suv', 'Tiger1', 'Toy', 'Trans',
'Twinnings', 'Vase'
]
dataset50_seq = [
'Basketball', 'Biker', 'Bird1', 'BlurBody', 'BlurCar2', 'BlurFace',
'BlurOwl', 'Bolt', 'Box', 'Car1', 'Car4', 'CarDark', 'CarScale',
'ClifBar', 'Couple', 'Crowds', 'Deer', 'Diving', 'DragonBaby', 'Dudek',
'Football', 'Freeman4', 'Girl', 'Human3', 'Human4', 'Human6', 'Human9',
'Ironman', 'Jump', 'Jumping', 'Liquor', 'Matrix', 'MotorRolling',
'Panda', 'RedTeam', 'Shaking', 'Singer2', 'Skating1', 'Skating2',
'Skiing', 'Soccer', 'Surfer', 'Sylvester', 'Tiger2', 'Trellis',
'Walking', 'Walking2', 'Woman'
]
if "OTB50" in dataset_path:
data_seq = dataset50_seq
else:
data_seq = dataset100_seq
log_name = 'log.txt'
log_file = open(log_name, 'w')
records_success = [] #defaultdict(list)
records_precision = [] #defaultdict(list)
records_reinit = defaultdict(list)
model_def = os.path.join(root_path, args.prototxt)
model_weight = os.path.join(root_path, args.caffemodel)
vggnet = VGGnet.VGGnet(model_def, model_weight)
thre_max_neg = 0.3 # 0.5
test_times = 1 # 0
for t in range(test_times):
print 'Test round: %d' % t
log_file.write('Test round: %d\n' % t)
# sequences = ['Fish']
for sequence in data_seq:
for t in os.walk(
os.path.join(dataset_path, sequence, sequence, "img")):
if t[0] == os.path.join(dataset_path, sequence, sequence,
"img"):
nFrame = len(t[2])
print 'Processing: %s' % sequence
log_file.write('Processing: %s\n' % sequence)
print "Total frames are: ", nFrame
log_file.write('Total frames are: %d\n' % nFrame)
gt_path = os.path.join(dataset_path, sequence, sequence,
"groundtruth_rect.txt")
gt_boxes = utils.get_boxes_all(gt_path)
conf_hist = []
iou_hist = []
area_hist = []
pred_hist = [] # (x1,y1,x2,y2)
eig_hist = []
reinit = 0
nFrame = np.minimum(nFrame, gt_boxes.shape[0])
id_shift = 0
init_id = False
for id in np.arange(0, nFrame):
frame_name = "img/%04d.jpg" % (id + 1)
# print "Start processing: %s" % frame_name
frame_path = os.path.join(dataset_path, sequence, sequence,
frame_name)
if os.path.exists(frame_path) == False:
id_shift = id_shift + 1
continue
id = id - id_shift
frame_data = caffe.io.load_image(
frame_path) # (432,576,3), in [0,1]
gt_box = gt_boxes[id]
if init_id == False:
h, w, c = frame_data.shape
frame_shape = [c, w, h]
fail_times = 0
area = (gt_box[2] - gt_box[0]) * (gt_box[3] - gt_box[1])
ratio = (gt_box[2] - gt_box[0]) / (gt_box[3] - gt_box[1]
) # ratio=w/h
# set up net.blobs['im_info']
print "Image Size: ", w, h
log_file.write('Image Size: %d %d\n' % (w, h))
vggnet.reshape(w=w, h=h, nbox=nparticles)
filter = PFfilter.PFfilter(
utils.bbox_to_states(gt_box, area, ratio), area, ratio,
w, h, nparticles)
filter.create_particles()
filter.restrict_particles(w, h)
area_hist.append(filter.cur_a)
pred_hist.append(np.array(gt_box).reshape(1, -1))
# pca
# test sample_iou
num_true = 500
num_false = 1000
boxes_train = []
# boxes_train_neg=[]
iou_train = []
try:
# Q=[[1,0],[0,1]] #for pixel wise
Q = 0.05 # box_w,box_h
sample_box_true, sample_iou_true = filter.sample_iou_pred_box(
gt_box, Q, 0.01, 0.01, num_true, 0.8, 1.0)
except OverflowError as e:
print "too many loops in sample."
# print sample_box_true[:10]
# print sample_box_true.shape[0]
# print sample_iou_true[:10]
# print "average iou: ", np.mean(sample_iou_true)
boxes_train.append(sample_box_true)
iou_train.append(sample_iou_true)
try:
# Q=[[36,0],[0,36]]#for pixel wise
Q = 0.2 # 0.15
sample_box_false, sample_iou_false = filter.sample_iou(
gt_box, Q, 0.2, 0.01, num_false / 2, 0,
thre_max_neg)
except OverflowError as e:
print "too many loops in sample."
# print sample_box_false[:10]
# print sample_box_false.shape[0]
# print sample_iou_false[:10]
# print "average iou: ", np.mean(sample_iou_false)
boxes_train.append(sample_box_false)
iou_train.append(sample_iou_false)
try:
# Q=[[36,0],[0,36]]#for pixel wise
Q = 0.2
sample_box_false, sample_iou_false = filter.sample_iou(
gt_box, Q, 0.01, 0.2, num_false / 2, 0,
thre_max_neg)
except OverflowError as e:
print "too many loops in sample."
# print sample_box_false[:10]
# print sample_box_false.shape[0]
# print sample_iou_false[:10]
# print "average iou: ", np.mean(sample_iou_false)
boxes_train.append(sample_box_false)
iou_train.append(sample_iou_false)
boxes_train = np.vstack(boxes_train)
iou_train = np.vstack(iou_train)
y_train_true = np.ones((num_true, ))
y_train_false = np.zeros((num_false, ))
y_train = np.hstack([y_train_true, y_train_false])
# permutation
ind_perm = np.random.permutation(
range(num_false + num_true))
boxes_train = boxes_train[ind_perm, :]
iou_train = iou_train[ind_perm]
y_train = y_train[ind_perm]
ind_pos = np.where(y_train == 1)[0]
ind_neg = np.where(y_train == 0)[0]
vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0])
features = vggnet.get_features_first_raw(
frame_data, boxes_raw=boxes_train, id=id)
for k, v in features.iteritems():
# print k,v.shape
if k == 'f3':
pca3 = utils.skl_pca(v)
v_pca3 = pca3.transform(v)
pca3_pos = np.zeros((num_true, pca3.n_components_),
dtype=np.float32)
pca3_neg = np.zeros(
(num_false, pca3.n_components_),
dtype=np.float32)
pca3_pos[...] = v_pca3[ind_pos, :]
pca3_neg[...] = v_pca3[ind_neg, :]
# utils.vis_as_image(v_pca3)
# plt.imshow(v_pca3)
# plt.title("PCA features")
# plt.show()
# plt.close()
# logistic regression
y_weight = sklearn.utils.class_weight.compute_class_weight(
class_weight='balanced',
classes=np.array([0, 1]),
y=y_train)
# print y_weight
class_weight = {0: y_weight[0], 1: y_weight[1]}
clf3 = linear_model.LogisticRegression(
fit_intercept=True, solver='liblinear')
clf3.fit(v_pca3, y_train)
vis_feature = False
if vis_feature:
utils.vis_features(features, id)
start_time = time.time()
else:
if fail_times >= 5:
# reinitialize
reinit += 1
area = (gt_box[2] - gt_box[0]) * (gt_box[3] -
gt_box[1])
ratio = (gt_box[2] - gt_box[0]) / (gt_box[3] -
gt_box[1])
filter = PFfilter.PFfilter(
utils.bbox_to_states(gt_box, area, ratio), area,
ratio, w, h, nparticles)
# filter.reset(utils.bbox_to_states(gt_box, area, ratio), area, ratio)
filter.create_particles()
filter.restrict_particles(w, h)
area_hist.append(filter.cur_a)
pred_box = gt_box
pred_hist.append(np.array(pred_box).reshape(1, -1))
conf_hist.append(-0.1)
boxes_train = []
# boxes_train_neg=[]
iou_train = []
try:
# Q=[[1,0],[0,1]] #for pixel wise
Q = 0.05 # box_w,box_h
sample_box_true, sample_iou_true = filter.sample_iou_pred_box(
gt_box, Q, 0.01, 0.01, num_true, 0.8, 1.0)
except OverflowError as e:
print "too many loops in sample."
boxes_train.append(sample_box_true)
iou_train.append(sample_iou_true)
try:
# Q=[[36,0],[0,36]]#for pixel wise
Q = 0.2 # 0.15
sample_box_false, sample_iou_false = filter.sample_iou(
gt_box, Q, 0.2, 0.01, num_false / 2, 0,
thre_max_neg)
except OverflowError as e:
print "too many loops in sample."
boxes_train.append(sample_box_false)
iou_train.append(sample_iou_false)
try:
# Q=[[36,0],[0,36]]#for pixel wise
Q = 0.2
sample_box_false, sample_iou_false = filter.sample_iou(
gt_box, Q, 0.01, 0.2, num_false / 2, 0,
thre_max_neg)
except OverflowError as e:
print "too many loops in sample."
# print sample_box_false[:10]
# print sample_box_false.shape[0]
# print sample_iou_false[:10]
# print "average iou: ", np.mean(sample_iou_false)
boxes_train.append(sample_box_false)
iou_train.append(sample_iou_false)
boxes_train = np.vstack(boxes_train)
iou_train = np.vstack(iou_train)
y_train_true = np.ones((num_true, ))
y_train_false = np.zeros((num_false, ))
y_train = np.hstack([y_train_true, y_train_false])
# permutation
ind_perm = np.random.permutation(
range(num_false + num_true))
boxes_train = boxes_train[ind_perm, :]
iou_train = iou_train[ind_perm]
y_train = y_train[ind_perm]
ind_pos = np.where(y_train == 1)[0]
ind_neg = np.where(y_train == 0)[0]
vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0])
features = vggnet.get_features_first_raw(
frame_data, boxes_raw=boxes_train, id=id)
for k, v in features.iteritems():
# print k, v.shape
if k == 'f3':
v_pca3 = pca3.transform(v)
pca3_pos[...] = v_pca3[ind_pos, :]
pca3_neg[...] = v_pca3[ind_neg, :]
clf3.fit(v_pca3, y_train)
score3 = clf3.score(v_pca3, y_train)
# print 'score3: ', score3
# prob=clf3.predict_proba(v_pca3)
# print clf3.classes_
fail_times = 0
continue
filter.predict_particles(Q=0.2, cr=0.005, ca=0.001)
filter.restrict_particles(w, h)
area_hist.append(filter.cur_a)
# compute conf
conf = np.zeros(filter.weights.shape)
# np.save('particles.npy',filter.particles)
pred_boxes = utils.state_to_bbox(filter.particles, area,
ratio)
vggnet.reshape(w, h, filter.num_particles)
features = vggnet.get_features_first_raw(
frame_data, boxes_raw=pred_boxes, id=id)
for k, v in features.iteritems():
# print k,v.shape
if k == 'f3':
v_pca3 = pca3.transform(v)
# utils.vis_as_image(v_pca3)
# plt.imshow(v_pca3)
# plt.title("PCA features")
# plt.show()
# plt.close()
# logistic regression
conf = clf3.predict_proba(v_pca3)[:, 1]
conf_max = np.max(conf)
conf_min = np.min(conf)
filter.update_particles(conf)
filter.resample() # always resample
pred_state, s_particles, r_particles = filter.estimate(
k=10)
cov_particles = np.dot(
filter.particles[:, :2].T,
filter.particles[:, :2]) / filter.particles.shape[0]
eigval, eigvec = np.linalg.eig(cov_particles)
max_val = eigval[0]
eig_hist.append(max_val)
print 'Max eigvalue: %f' % max_val
# print 'conf is: ',conf
if conf_max > 0.8:
fail_times = 0
else:
fail_times += 1
print "conf_max too low, not update particles "
pred_box = utils.state_to_bbox(pred_state.reshape((-1, 6)),
area, ratio)
show_sr = False
if show_sr:
count, xedge, yedge, tmp_im = plt.hist2d(
s_particles,
r_particles,
bins=10,
weights=filter.weights.squeeze(),
cmap=plt.cm.gray)
top3 = np.argsort(-count, axis=None)[:3]
row_ind = top3[:] / count.shape[1]
col_ind = top3[:] % count.shape[0]
'''
plt.scatter(s_particles,r_particles,c='r',marker='.',linewidths=1)
plt.xlabel('Area')
plt.ylabel('Aspect ratio')
plt.title('Area and Ratio of particles')
plt.axis('equal')
'''
plt.show()
iou = utils.calc_iou(gt_box, pred_box)
# print 'iou is: ', iou
pred_hist.append(pred_box)
conf_hist.append(conf_max)
iou_hist.append(iou)
if conf_max >= 0.9: # 0.8
# update pca3_pos and pca3_neg
new_true = 100 # 50
new_false = 200 # 100
boxes_train = []
iou_train = []
Q = 0.02
try:
sample_box_true, sample_iou_true = filter.sample_iou_pred_box(
pred_box, Q, 0.01, 0.01, new_true, 0.85, 1.0)
except OverflowError as e:
print "too many loops in sample."
# print sample_box_true[:10]
# print sample_box_true.shape[0]
# print sample_iou_true[:10]
# print "average iou: ", np.mean(sample_iou_true)
boxes_train.append(sample_box_true)
iou_train.append(sample_iou_true)
try:
Q = 0.2
sample_box_false, sample_iou_false = filter.sample_iou(
pred_box, Q, 0.2, 0.01, new_false / 2, 0,
thre_max_neg)
except OverflowError as e:
print "too many loops in sample."
# print sample_box_false[:10]
# print sample_box_false.shape[0]
# print sample_iou_false[:10]
# print "average iou: ", np.mean(sample_iou_false)
boxes_train.append(sample_box_false)
iou_train.append(sample_iou_false)
try:
Q = 0.2
sample_box_false, sample_iou_false = filter.sample_iou(
pred_box, Q, 0.01, 0.2, new_false / 2, 0,
thre_max_neg)
except OverflowError as e:
print "too many loops in sample."
boxes_train.append(sample_box_false)
iou_train.append(sample_iou_false)
boxes_train = np.vstack(boxes_train)
iou_train = np.vstack(iou_train)
y_train_true = np.ones((new_true, ))
y_train_false = np.zeros((new_false, ))
y_train = np.hstack([y_train_true, y_train_false])
# permutation
ind_perm = np.random.permutation(
range(new_false + new_true))
boxes_train = boxes_train[ind_perm, :]
y_train = y_train[ind_perm]
new_y = np.zeros(y_train.shape)
new_y[...] = y_train
ind_pos = np.where(y_train == 1)[0]
ind_neg = np.where(y_train == 0)[0]
vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0])
features = vggnet.get_features_second_raw(
boxes_raw=boxes_train, id=id)
for k, v in features.iteritems():
# print k, v.shape
if k == 'f3':
v_pca3 = pca3.transform(v)
# random substitude
pca3_cur_pos = v_pca3[ind_pos, :]
pca3_cur_neg = v_pca3[ind_neg, :]
to_subst = random.sample(
range(num_true), new_true)
pca3_pos[to_subst, :] = pca3_cur_pos
to_subst = random.sample(
range(num_false), new_false)
pca3_neg[to_subst, :] = pca3_cur_neg
if conf_max < 1 and fail_times >= 2:
pca3_train = np.vstack([pca3_pos, pca3_neg])
y_train_true = np.ones((num_true, ))
y_train_false = np.zeros((num_false, ))
y_train = np.hstack([y_train_true, y_train_false])
# permutation
ind_perm = np.random.permutation(
range(num_false + num_true))
pca3_train = pca3_train[ind_perm, :]
y_train = y_train[ind_perm]
# logistic regression
clf3.fit(pca3_train, y_train)
# print 'score is: ',clf3.score(pca3_train,y_train)
# (B,G,R)
frame_data_cv = frame_data * 255 # [0,1]-->[0,255]
frame_data_cv = frame_data_cv[:, :, ::-1] # RGB->BGR
frame_data_cv = frame_data_cv.astype('uint8')
cv2.rectangle(frame_data_cv, (int(gt_box[0]), int(gt_box[1])),
(int(gt_box[2]), int(gt_box[3])), (255, 0, 0), 2,
1)
if id > 0 and init_id == True:
cv2.rectangle(frame_data_cv,
(int(pred_box[0, 0]), int(pred_box[0, 1])),
(int(pred_box[0, 2]), int(pred_box[0, 3])),
(0, 255, 0), 2, 1)
if init_id == False:
init_id = True
show_particles = False
if show_particles:
for i in range(filter.num_particles):
cx = filter.particles[i, 0]
cy = filter.particles[i, 1]
cv2.circle(frame_data_cv, (int(cx), int(cy)),
1, (0, 0, 255),
thickness=1)
show_box = False
if show_box:
n = 0
for i in ind_pos:
if n % 5 == 0:
cv2.rectangle(frame_data_cv, (int(
boxes_train[i, 0]), int(boxes_train[i, 1])),
(int(boxes_train[i, 2]),
int(boxes_train[i, 3])),
(0, 0, 255), 2, 1)
n += 1
n = 0
'''
for i in ind_neg:
if n%15==0:
cv2.rectangle(frame_data_cv, (int(boxes_train[i, 0]), int(boxes_train[i, 1])),
(int(boxes_train[i, 2]), int(boxes_train[i, 3])), (0, 255,255), 2, 1)
n+=1
'''
show_particles_init = False
if show_particles_init:
for i in range(filter.num_particles):
cx = filter.particles[i, 0]
cy = filter.particles[i, 1]
cv2.circle(frame_data_cv, (int(cx), int(cy)),
1, (0, 255, 0),
thickness=1)
show_frame = False
cv2.circle(frame_data_cv,
(int(filter.cur_c[0]), int(filter.cur_c[1])),
2, (0, 0, 255),
thickness=1)
if show_frame:
cv2.imshow(sequence, frame_data_cv)
c = cv2.waitKey(1)
# print 'You press: ',chr(c)
# if chr(c)=='c':
if c != -1:
cv2.destroyWindow(sequence)
# conf_hist=np.array(conf_hist)
# iou_hist=np.array(iou_hist)
# np.save('conf_hist.npy',conf_hist)
# np.save('iou_hist.npy',iou_hist)
break
end_time = time.time()
print "Average FPS: %f" % (nFrame / (end_time - start_time))
log_file.write("Average FPS: %f\n" % (nFrame /
(end_time - start_time)))
conf_hist = np.array(conf_hist)
iou_hist = np.array(iou_hist)
area_hist = np.array(area_hist)
pred_hist = np.vstack(pred_hist)
precisions, auc_pre = utils.calc_prec(gt_boxes, pred_hist)
# plt.figure()
# plt.subplot(221)
# plt.plot(precisions)
# plt.gca().invert_xaxis()
# plt.title("Precision plot")
# plt.xlabel('Location error threshold')
# plt.ylabel('Precision')
# plt.yticks(np.linspace(0,1,11))
# plt.subplot(222)
# plt.show()
suc, auc_iou = utils.calc_success(iou_hist)
records_precision.append(precisions * nFrame)
records_success.append(suc * nFrame)
# plt.plot(suc)
# plt.gca().invert_xaxis()
# plt.title('Success plot')
# plt.xlabel('Overlap threshold')
# plt.ylabel('Success Rate')
# plt.yticks(np.linspace(0,1,11))
# plt.show()
# np.save('conf_hist.npy', conf_hist)
# np.save('iou_hist.npy', iou_hist)
# np.save('area_hist.npy',area_hist)
# print 'Average iou is: %f'%(np.mean(iou_hist))
print 'Precision @20 is: %f' % precisions[19]
print 'Auc of Precision is: %f' % auc_pre
print 'Auc of Success is: %f' % auc_iou
print 'Reinit times: %d' % reinit
log_file.write("Precision @20 is: %f\n" % precisions[19])
log_file.write('Auc of Precision is: %f\n' % auc_pre)
log_file.write('Auc of Success is: %f\n' % auc_iou)
log_file.write('Reinit times: %d\n' % reinit)
log_file.close()
pkl = open(
'/home/ccjiang/Documents/caffe-fast-rcnn/examples/tracker/results_100.pkl',
'w')
pickle.dump([records_precision, records_success], pkl)
pkl.close()
