def rot_box(box,cterxy,imgwh,rot_angle,scale=1.0,correction=True): ''' Args: box:边框坐标[xmin,ymin,xmax,ymax] cterxy:旋转中心点坐标 [cter_x,cter_y] imgwh:图片宽高[w,h] rot_angle:旋转角 scale:放缩尺度 correction: bool,修正旋转后的目标框为正常左上右下坐标 return: box:边框坐标[x1,y1,x2,y2,x3,y3,x4,y4],左上开始,逆时针 ''' result_box = [] xmin,ymin,xmax,ymax = box[0],box[1],box[2],box[3] complete_coords = [xmin,ymin,xmin,ymax,xmax,ymax,xmax,ymin] for i in range(int(len(complete_coords)/2)): rotx,roty = rot_xy(cterxy[0],cterxy[1],complete_coords[2*i],complete_coords[2*i+1],rot_angle,scale) result_box.append(rotx) result_box.append(roty) if correction: xmin = min(result_box[0:len(result_box):2]) xmax = max(result_box[0:len(result_box):2]) ymin = min(result_box[1:len(result_box):2]) ymax = max(result_box[1:len(result_box):2]) xmin_v = utils.confine(xmin,0,imgwh[0]-1) ymin_v = utils.confine(ymin,0,imgwh[1]-1) xmax_v = utils.confine(xmax,0,imgwh[0]-1) ymax_v = utils.confine(ymax,0,imgwh[1]-1) #使用阈值剔除边缘截断严重的目标 if utils.calc_iou([xmin,ymin,xmax,ymax],[xmin_v,ymin_v,xmax_v,ymax_v]) < 0.5: xmin_v,ymin_v,xmax_v,ymin_v=0,0,0,0 return [xmin_v,ymin_v,xmin_v,ymax_v,xmax_v,ymax_v,xmax_v,ymin_v] else: return complete_coords
def sample_area(self, pred_box1, num, thre_max=0.5, thre_min=0): # sample area pred_box = pred_box1.squeeze() pred_box_w = pred_box[2] - pred_box[0] pred_box_h = pred_box[3] - pred_box[1] pred_box_cx = pred_box[0] + pred_box_w / 2.0 pred_box_cy = pred_box[1] + pred_box_h / 2.0 samples = np.ones((num, 4), dtype=np.float32) samples_wh = np.ones((num, 2), dtype=np.float32) # samples_wh[:,0]*=pred_box_w # samples_wh[:,1]*=pred_box_h cur_n = 0 # s:(0.3,0.7664) s = 0.23 + np.random.chisquare(3, num) * 0.09 s = 1 - s ind = np.where(s > 0.3)[0] s = s[ind[:num / 2]] samples_wh[:num / 2, 0] = pred_box_w * s samples_wh[:num / 2, 1] = pred_box_h * s # s:(1.3258,2) s = 1.325 + np.random.chisquare(3, num) * 0.125 ind = np.where(s <= 2)[0] s = s[ind[:num / 2]] samples_wh[num / 2:, 0] = pred_box_w * s samples_wh[num / 2:, 1] = pred_box_h * s # (x_c,y_c)-->(x1,y1,x2,y2) samples[:, 0] = pred_box_cx - samples_wh[:, 0] / 2.0 samples[:, 1] = pred_box_cy - samples_wh[:, 1] / 2.0 samples[:, 2] = pred_box_cx + samples_wh[:, 0] / 2.0 samples[:, 3] = pred_box_cy + samples_wh[:, 1] / 2.0 samples = utils.restrict_box(samples, self.width, self.height) iou = utils.calc_iou(pred_box, samples) iou = iou[:, np.newaxis] return samples, iou
def nms_box(preds, confm, feat, conf): # preds:(N,4),(x1,y1,x2,y2) # N=preds.shape[0] res_box = [] res_feat = [] res_conf = [] thresh = 0.5 # 0.7 t = [] while preds.shape[0] > 0: ind = np.argsort(-confm.squeeze()).tolist() i = ind[0] cur_box = preds[i] cur_iou = utils.calc_iou(cur_box, preds) i_spa = np.where(cur_iou < thresh)[0] res_box.append(cur_box) res_feat.append(feat[i]) res_conf.append(conf[i, :]) t.append(preds.shape[0] - i_spa.shape[0]) preds = preds[i_spa] confm = confm[i_spa] feat = feat[i_spa] conf = conf[i_spa] return np.array(res_box), np.array(res_feat), np.array(res_conf), np.array( t)
def toi_labeling( self, toi_boxes, entities, mode, ): ious = calc_iou(toi_boxes, np.array([(e[0], e[1]) for e in entities])) max_ious = ious.max(axis=1) max_idx = ious.argmax(axis=1) tois = [] for i in range(toi_boxes.shape[0]): if max_ious[i] == 1: self.candidation_hit[mode] += 1 tois.append( (toi_boxes[i, 0], toi_boxes[i, 1], entities[max_idx[i]][2])) elif max_ious[i] >= 1 and mode == "train": tois.append( (toi_boxes[i, 0], toi_boxes[i, 1], entities[max_idx[i]][2])) elif max_ious[i] < self.config.train_neg_iou_th or mode != "train": tois.append((toi_boxes[i, 0], toi_boxes[i, 1], 0)) self.tois_num[mode] += (len(tois)) return sorted(tois)
def generate_patch(boxes, threshold): """ Function to generate a random patch within the image If the patch overlaps any gt boxes at above the threshold, then the patch is picked, otherwise generate another patch Args: boxes: box tensor (num_boxes, 4) threshold: iou threshold to decide whether to choose the patch Returns: patch: the picked patch ious: an array to store IOUs of the patch and all gt boxes """ while True: patch_w = random.uniform(0.1, 1) scale = random.uniform(0.5, 2) patch_h = patch_w * scale patch_xmin = random.uniform(0, 1 - patch_w) patch_ymin = random.uniform(0, 1 - patch_h) patch_xmax = patch_xmin + patch_w patch_ymax = patch_ymin + patch_h patch = np.array([[patch_xmin, patch_ymin, patch_xmax, patch_ymax]], dtype=np.float32) patch = np.clip(patch, 0.0, 1.0) ious = calc_iou(tf.constant(patch), boxes) if tf.math.reduce_any(ious >= threshold): break return patch[0], ious[0]
def counter_for_actual_accuracy(pred_labels, target_labels): #find position for eval_actual #given list of targets and predictions, find which prediction corresponds to which target. iou_choice_counter = 0 mega_iou_choice = [] for key, pred_label_list in pred_labels.items(): if key in target_labels: iou_list = [] target_label_list = target_labels[key] for target_idx, target_label in enumerate(target_label_list): for pred_idx, pred_label in enumerate(pred_label_list): iou_val = utils.calc_iou(pred_label, target_label) iou_list.append([ iou_val, pred_idx, target_idx, pred_label, target_label ]) list_len = min(len(target_label_list), len(pred_label_list)) iou_list = sorted(iou_list, key=lambda k: (k[0], random.random()), reverse=True) iou_choice = [] while len(iou_list) != 0 and len(iou_choice) < list_len: if len(iou_choice) == 0: iou_choice.append(iou_list.pop(0)) else: #pdb.set_trace() cur_item = iou_list.pop(0) flag = True for item in iou_choice: if cur_item[1] == item[1]: flag = False break if cur_item[2] == item[2]: flag = False break if flag: iou_choice.append(cur_item) mega_iou_choice.extend(iou_choice) #============================================================================================ #for actual accuracy: check if center of prediction is within (start, end) boundaries of target for item in mega_iou_choice: iou_val, pred_idx, target_idx, pred_label, target_label = item pred_start, pred_end = pred_label target_start, target_end = target_label center_pred = float(pred_end + pred_start) / 2 if round(center_pred, 2) >= round(target_start, 2) and round( center_pred, 2) <= round(target_end, 2): iou_choice_counter += 1 return iou_choice_counter
def form_gt(self, gt_bboxes: List[torch.Tensor]) -> Tuple[torch.Tensor, torch.Tensor]: """ Builds 4D torch.Tensor with a shape of the detector output for the batch of frames. :param gt_bboxes: list of torch.Tensors of shape [n_bboxes, n_bbox_params] of ground truth bounding boxes parameters which are center coordinates, length, width, sin(a) and cos(a) :return: Tuple of two 4D torch.Tensor of ground truth data: First tensor is of shape [batch_size, detector_out_width, detector_out_length, n_predefined_boxes] and represents classification target with ones for boxes with associated GT boxes Second tensor is of shape [batch_size, detector_out_width, detector_out_length, n_predefined_boxes, n_bbox_params] and represents regression target """ gt_bboxes_coords = [utils.bbox_to_coordinates(gt_bbox, rot=False).to(self.device) for gt_bbox in gt_bboxes] classification_target = [] regression_target = [] prior_boxes_params = self.prior_boxes_params.view(-1, self.n_bbox_params) prior_boxes_coords = self.prior_boxes_coords.view(-1, 4, 2) # each box is 4 points of 2 coordinates) for gt_bbox, gt_bbox_coords in zip(gt_bboxes, gt_bboxes_coords): iou = utils.calc_iou(gt_bbox_coords, prior_boxes_coords) # tensor of size [n_gt_boxes, n_prior_boxes] gt_has_match = (iou >= self.iou_threshold).sum(dim=1).bool() prior_has_match = (iou >= self.iou_threshold).sum(dim=0).bool() gt_best_match = iou.argmax(dim=1) prior_match = iou.argmax(dim=0) prior_match[gt_best_match[~gt_has_match]] = (~gt_has_match).nonzero().flatten() prior_has_match[gt_best_match[~gt_has_match]] = True prior_has_match = prior_has_match.view( self.detector_out_width, self.detector_out_length, len(self.predefined_bboxes) ).float() cur_regression_target = torch.zeros(self.detector_out_width * self.detector_out_length * len(self.predefined_bboxes), self.n_bbox_params, device=self.device) # Calculate regression target: y and x offset, width and length correction, sin and cos of angle cur_regression_target[:, 0] = \ (prior_boxes_params[:, 0] - gt_bbox[prior_match, 0]) / gt_bbox[prior_match, 2] cur_regression_target[:, 1] = \ (prior_boxes_params[:, 1] - gt_bbox[prior_match, 1]) / gt_bbox[prior_match, 3] cur_regression_target[:, 2] = torch.log(prior_boxes_params[:, 2] / gt_bbox[prior_match, 2]) cur_regression_target[:, 3] = torch.log(prior_boxes_params[:, 3] / gt_bbox[prior_match, 3]) cur_regression_target[:, 4] = gt_bbox[prior_match, 4] cur_regression_target[:, 5] = gt_bbox[prior_match, 5] cur_regression_target = \ cur_regression_target.view(self.detector_out_width, self.detector_out_length, len(self.predefined_bboxes), self.n_bbox_params) cur_regression_target *= prior_has_match.unsqueeze(3) regression_target.append(cur_regression_target) classification_target.append(prior_has_match) return torch.stack(classification_target), torch.stack(regression_target)
def sample_xy(self, pred_box1, num, thre_max=0.5, thre_min=0.0): # only perform xy shift # x alone pred_box = pred_box1.squeeze() alone = 1 / 3.0 both = 1 - np.sqrt(2 / 3.0) pred_box_w = pred_box[2] - pred_box[0] pred_box_h = pred_box[3] - pred_box[1] pred_box_cx = pred_box[0] + pred_box_w / 2.0 pred_box_cy = pred_box[1] + pred_box_h / 2.0 samples = np.ones((num, 4), dtype=np.float32) samples_c = np.ones((num, 2), dtype=np.float32) samples_c[:, 0] *= pred_box_cx samples_c[:, 1] *= pred_box_cy num1 = num / 2 # +-dx,+-dy dx = 0.25 + np.random.chisquare(3, num1) * 0.1 samples_c[:num1 / 4, 0] += (pred_box_w * dx[:num1 / 4]) # dx samples_c[num1 / 4:num1 / 2, 0] -= (pred_box_w * dx[num1 / 4:num1 / 2]) # -dx samples_c[num1 / 2:3 * num1 / 4, 1] += (pred_box_h * dx[num1 / 2:num1 * 3 / 4]) # dy samples_c[3 * num1 / 4:num1, 1] -= (pred_box_h * dx[3 * num1 / 4:]) # -dy # |dx2|=|dy2| dx2 = 0.15 + np.random.chisquare(3, num) * 0.03 # dx,dy samples_c[num1:(num1 + num1 / 4), 0] += (pred_box_w * dx2[:num1 / 4]) samples_c[num1:(num1 + num1 / 4), 1] += (pred_box_h * (dx2[num1 / 4:num1 / 2])) # -dx,dy samples_c[(num1 + num1 / 4):(num1 + num1 / 2), 0] -= (pred_box_w * dx2[num1 / 2:num1 * 3 / 4]) samples_c[(num1 + num1 / 4):(num1 + num1 / 2), 1] += (pred_box_h * (dx2[3 * num1 / 4:num1])) # dx,-dy samples_c[(num1 + num1 / 2):(num1 + 3 * num1 / 4), 0] += (pred_box_w * dx2[num1:(num1 + num1 / 4)]) samples_c[(num1 + num1 / 2):(num1 + 3 * num1 / 4), 1] -= ( pred_box_h * (dx2[(num1 + num1 / 4):(num1 + num1 / 2)])) # -dx,-dy samples_c[(num1 + 3 * num1 / 4):, 0] -= (pred_box_w * dx2[(num1 + num1 / 2):(num1 + 3 * num1 / 4)]) samples_c[(num1 + 3 * num1 / 4):, 1] -= ( pred_box_h * (dx2[(num1 + 3 * num1 / 4):])) # (x_c,y_c)-->(x1,y1,x2,y2) samples[:, 0] = samples_c[:, 0] - pred_box_w / 2.0 samples[:, 1] = samples_c[:, 1] - pred_box_h / 2.0 samples[:, 2] = samples_c[:, 0] + pred_box_w / 2.0 samples[:, 3] = samples_c[:, 1] + pred_box_h / 2.0 samples = utils.restrict_box(samples, self.width, self.height) iou = utils.calc_iou(pred_box, samples) iou = iou[:, np.newaxis] return samples, iou
def nms_pred(pred,preds, vpca3,conf): # preds:(N,4),(x1,y1,x2,y2) # N=preds.shape[0] thresh = 0.3 # 0.7 cur_iou = utils.calc_iou(pred, preds) i_spa = np.where(cur_iou < thresh)[0] i_spa2 = np.where(conf > 0.4)[0] print "IOU: ",i_spa print "Conf: ",i_spa2 ind = np.array(list(set(i_spa).intersection(set(i_spa2)))) print ind if ind.shape[0] > 0: res_box = preds[ind,:] res_vpca3 = vpca3[ind,:] return True,res_vpca3 else: return False,np.zeros((1,4))
def crop_box(box, top_left_x, top_left_y, crop_w, crop_h, iou_thr=0.5): '''目标框坐标平移变换 Args: box:目标框坐标[xmin,ymin,xmax,ymax] top_left_x,top_left_y:裁剪图像左上角坐标 crop_w,crop_h:裁剪部分图像宽高 iou_thr: iou阈值,去除裁剪后过小目标 return: crop_box:平移变换结果[xmin,ymin,xmax,ymax] ''' xmin, ymin = crop_xy(box[0], box[1], top_left_x, top_left_y, crop_w, crop_h) xmax, ymax = crop_xy(box[2], box[3], top_left_x, top_left_y, crop_w, crop_h) croped_box = [xmin, ymin, xmax, ymax] if utils.calc_iou([0, 0, box[2] - box[0], box[3] - box[1]], [0, 0, xmax - xmin, ymax - ymin]) < iou_thr: croped_box = [0, 0, 0, 0] return croped_box
def evaluate_test_dataset(device, exp, dataset, N=5): print(f'******* {dataset}') miou = [] path = os.path.join(opt.dataroot, dataset, 'experiments', exp) model_type = 101 if 'ResNet101' in exp else 50 for i in range(N): model = load_model(device, model_type, path, i) test_loader = load_test_dataset() iou = [] i = 0 for X, y in test_loader: X = X.to(device) y = y.to(device) output = model(X)['out'] probs = torch.functional.F.softmax(output, 1) y_pred = torch.argmax(probs, dim=1) iou.append(calc_iou(y_pred, y)) i += 1 if i == 76: break miou.append(np.mean(iou)) print(miou[-1]) print(miou) print(np.mean(miou), np.std(miou))
def train(): assert max_point == 8192, "wrong max point,need to change pointnet_util->fp1 = PointNetFeaturePropagation_PointConv->in_channel" # read device device = torch.device("cuda" if torch.cuda.is_available() else "cpu") print("device: {}".format(device), end="\n") # load dataset data_loader_train = Scannetloader(scannet_file_root, max_point, n_class, is_train=True) data_loader_val = Scannetloader(scannet_file_root, max_point, n_class, is_train=False) train_loader = torch.utils.data.DataLoader(data_loader_train, batch_size=batch_size, shuffle=True, num_workers=n_workers) val_loader = torch.utils.data.DataLoader(data_loader_val, batch_size=batch_size, shuffle=True, num_workers=n_workers) # set model running devices #model = PointNet2SemSeg(n_class) model = PointNet2SemSeg_PointConv(n_class) model.to(device) #?torch.backends.cudnn.benchmark = True print("usable gpu nums: {}".format(torch.cuda.device_count())) # set optimizer, lr_scheduler, loss function optimizer = None if optimizer_name == "rmsprop": optimizer = torch.optim.RMSprop(model.parameters(), lr=lr, momentum=momentum, weight_decay=decay_rate) elif optimizer_name == "sgd": optimizer = torch.optim.SGD(model.parameters(), lr=lr, momentum=momentum, weight_decay=decay_rate) elif optimizer_name == "adam": optimizer = torch.optim.Adam(model.parameters(), lr=lr, betas=(0.9, 0.999), eps=1e-08) lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=40, gamma=0.5) # criterion #criterion = torch.nn.BCEWithLogitsLoss() #criterion = F.nll_loss() model, optimizer = amp.initialize(model, optimizer, opt_level="O1") # O1 not 01 model = torch.nn.DataParallel( model, device_ids=[i for i in range(torch.cuda.device_count())]) # load checkpoints last_best_iou = -100. load_ckpt_path = os.path.join("checkpoints", load_ckpt_name) start_epoch = 0 if is_load_checkpoints: if torch.cuda.is_available(): checkpoint = torch.load(load_ckpt_path) else: checkpoint = torch.load(load_ckpt_path, map_location='cpu') model.load_state_dict(checkpoint['model_state']) last_best_iou = checkpoint['best_iou'] start_epoch = checkpoint['epoch'] optimizer.load_state_dict(checkpoint["optimizer"]) amp.load_state_dict(checkpoint["amp"]) print('Checkpoint resume success... last iou:{:.4%},last epoch:{}'. format(last_best_iou, start_epoch)) # train epoch best_iou = last_best_iou time_epoch = time.time() i = 0 print("training...") for epoch in range(start_epoch, max_epoch): time_step = time.time() lr_scheduler.step(epoch=epoch) for step, batch in enumerate(train_loader): model.train() points = batch[0].to(device) targets = batch[1].long().to(device) points = points.transpose(2, 1) optimizer.zero_grad() # print(points.shape) = [3, 9, 8192] pred = model(points[:, :3, :], points[:, 3:, :]) pred = pred.contiguous().view(-1, n_class) # targets = targets.view(-1, 1)[:, 0] targets = targets.contiguous().view(-1) #loss = criterion(pred.float(),targets.float()) loss = F.cross_entropy(input=pred, target=targets) # log_softmax + nll_loss # loss = F.nll_loss(pred, targets) # softmax + Cross-Entropy cost cur_loss = np.float(loss.cpu().detach().numpy( )) # 不要用.data,会导致tensor flag: requires_grad=False,终止求导 with amp.scale_loss(loss, optimizer) as scaled_loss: scaled_loss.backward() #loss.backward() optimizer.step() writer.add_scalars('Train_record/Loss per X step', {'loss': cur_loss}, step + epoch * len(train_loader)) cur_time = time.time() print( "\rtrain | epoch: {}/{} | step:{}/{} | {} s/step | time used:{:d}s | currency loss:{:.4}" .format(epoch, max_epoch, step + 1, len(train_loader), int((cur_time - time_step) / (step + 1)), int(cur_time - time_epoch), cur_loss), end='', flush=True) # val print("\nvaluing") with torch.no_grad(): model.eval() cat_table = 0 total_cat_table = np.zeros((2, n_class)) # [0]:acc,[1]:miou time_step_val = time.time() for step_val, batch_val in enumerate(val_loader): #print("read batch time used:",time.time()-time_step) points_val = batch_val[0].to(device) labels_val = batch_val[1].cpu().data.numpy() points_val = points_val.transpose(2, 1) pred_val = model(points_val[:, :3, :], points_val[:, 3:, :]) # pred_val.shape = [batch_size,n_point,n_class] pred_val = F.softmax(pred_val, dim=2).cpu().data.numpy() pred_val = np.argmax(pred_val, axis=2) #visualizer(points_val.transpose(2,1)[0],pred_val[0],labels_val[0]) cat_table = calc_iou(pred_val, labels_val) total_cat_table += cat_table cur_time = time.time() print( "\rvalue | step:{}/{} | {} s/step | mean iou:{:.4} | mean acc:{:.4}" .format(step_val + 1, len(val_loader), int((cur_time - time_step_val) / (step_val + 1)), np.mean(cat_table[1]), np.mean(cat_table[0])), end='', flush=True) total_cat_table = total_cat_table / len(val_loader) # total_cat_table = total_cat_table / (step_val+1) mean_iou = np.mean(total_cat_table[1]) mean_acc = np.mean(total_cat_table[0]) print("\ntrue accuracy:{:.3f}".format( calc_acc(pred_val, labels_val))) print("\naccuracy: {:.3f} | mean iou: {:.3f}".format( mean_acc, mean_iou)) writer.add_scalars('Val_record/mIoU', {'accuracy': mean_acc}, epoch) writer.add_scalars('val_record/mIoU', {'mIoU': mean_iou}, epoch) print("class name \tmean accuracy\tmIoU") for i in range(n_class): print("%-18s %.3f \t%.3f" % (CLASS_LABELS[i], total_cat_table[0][i], total_cat_table[1][i])) if mean_iou > best_iou: best_iou = mean_iou state = { "epoch": epoch + 1, "model_state": model.state_dict(), "best_iou": best_iou, "amp": amp.state_dict(), "optimizer": optimizer.state_dict() } if not os.path.isdir('./checkpoints'): os.mkdir('./checkpoints') save_path = os.path.join( './checkpoints', "apex_eph_{}_iou_{:.2%}.ckpt.pth".format( epoch + 1, best_iou)) torch.save(state, save_path) print("checkpoint saved success") writer.close()
def sample_iou_new(self, gt_box, Q, T, R, N, thre_min=0, thre_max=1): sample_boxN = [] sample_iouN = [] cur_n = 0 D = np.array([[self.box_w, 0], [0, self.box_h]]) QL = D * Q # QL = np.linalg.cholesky(Q) a = 1.5 # 1.5 gt_state = utils.bbox_to_states(gt_box, self.area, self.ratio) sample_times = 0 chg_i = 0 while cur_n < N: sample_particles = np.zeros((N, 6), dtype=np.float32) QL = D * Q sample_particles[:, :2] = gt_state[:, :2] + np.dot(QL, np.random.randn(2, N)).transpose() if self.mltply: dsn = np.random.randn(N) * T ds = np.power(a, dsn) else: ds = 1 + np.random.randn(N) * T ds = np.maximum(0.01, ds) # in case of ds<0 sample_particles[:, 2] = gt_state[:, 2] * ds if self.mltply: dr = np.random.randn(N) * R + 1 else: dr = 1 + np.random.randn(N) * R dr = np.maximum(0.01, dr) # in case of dr<0 sample_particles[:, 3] = gt_state[:, 3] * dr # get box if self.mltply: sample_box = utils.state_to_bbox_m(sample_particles, self.area, self.ratio) else: sample_box = utils.state_to_bbox(sample_particles, self.area, self.ratio) sample_box = utils.restrict_box(sample_box, self.width, self.height) # compute iou sample_iou = utils.calc_iou(gt_box, sample_box) # restrict iou ind = np.where((sample_iou >= thre_min) & (sample_iou <= thre_max)) sample_box = sample_box[ind[0]] sample_iou = sample_iou[ind[0]] cur_n += sample_box.shape[0] sample_boxN.append(sample_box) sample_iouN.append(sample_iou.reshape((-1, 1))) if ind[0].shape[0] < N / 2 and thre_max >= 0.8: if chg_i == 0: Q *= 0.5 chg_i = (chg_i + 1) % 3 else: if chg_i == 1: T *= 0.5 T = np.minimum(T, 0.5) chg_i = (chg_i + 1) % 3 else: R *= 0.5 R = np.minimum(R, 0.5) chg_i = (chg_i + 1) % 3 if ind[0].shape[0] < N / 2 and thre_min <= 0.5: if chg_i == 0: Q *= 2 chg_i = (chg_i + 1) % 3 else: if chg_i == 1: T *= 2 T = np.minimum(T, 0.5) chg_i = (chg_i + 1) % 3 else: R *= 2 R = np.minimum(R, 0.5) chg_i = (chg_i + 1) % 3 sample_times += 1 if sample_times >= 100: # and cur_n>N/2.0:#100 # print "Caution: too many loops in sampling" # break raise OverflowError() if cur_n >= N: sample_boxN = np.vstack(sample_boxN)[:N, :] sample_iouN = np.vstack(sample_iouN)[:N, :] else: diff_n = N - cur_n sample_iouN = np.vstack(sample_iouN) sample_boxN = np.vstack(sample_boxN) diff_ind = random.sample(range(cur_n), diff_n) # need to ensure diff_n<cur_n sample_boxN = np.vstack([sample_boxN, sample_boxN[diff_ind]]) sample_iouN = np.vstack([sample_iouN, sample_iouN[diff_ind]]) return sample_boxN, sample_iouN
for i in range(filter.num_particles): cx = filter.particles[i, 0] cy = filter.particles[i, 1] cv2.circle(frame_data, (int(cx), int(cy)), 1, (0, 0, 255), thickness=1) # compute conf conf = np.zeros(filter.weights.shape) # np.save('particles.npy',filter.particles) pred_boxes = utils.state_to_bbox(filter.particles, area, ratio) # print 'pred_boxes: ',pred_boxes # for i in range(conf.shape[0]): # pred_box=pred_boxes[i,:] # print "pred_box is: ",pred_box # conf[i]=np.dot(gt_box,pred_box)/np.linalg.norm(gt_box,ord=2) # conf[i]=np.dot(gt_box,pred_box)/np.sum(np.square(gt_box)) conf = utils.calc_iou(gt_box, pred_boxes) # print 'conf is: ',conf filter.update_particles(conf) if filter.neff() < len(filter.particles): # 1/2 filter.resample() pred_state = filter.estimate() pred_box = utils.state_to_bbox(pred_state.reshape((-1, 6)), area, ratio) print 'ground truth bbox is: ', gt_box print "pred_box is: ", pred_box iou = utils.calc_iou(gt_box, pred_box) print 'iou is: ', iou # (B,G,R) show_frame = True cv2.circle(frame_data, (int(filter.cur_pos[0, 0]), int(filter.cur_pos[0, 1])), 2, (0, 0, 255), thickness=1) if show_frame: cv2.imshow(sequence, frame_data)
#print('Begin to train model_rpn') #test #print(np.shape(X)) #print(np.shape(Y[0]),' ',np.shape(Y[1])) #P_rpn = model_rpn.predict_on_batch(X) #print(np.shape(P_rpn[0]),' ',np.shape(P_rpn[1])) #endtest loss_rpn = model_rpn.train_on_batch(X, Y) #训练model_rpn #print('Predict rpn') P_rpn = model_rpn.predict_on_batch(X) #测试rpn #R:boxes, probs 返回经过npm后剩下的bbox以及对应的probs ((左上,右下坐标),序号) (anchors, 序号 ) R = rpn_to_roi(P_rpn[0], P_rpn[1], use_regr=True, overlap_thresh=0.7, max_boxes=300) # note: calc_iou converts from (x1,y1,x2,y2) to (x,y,w,h) format #X:[[x1, y1, w, h],...] Y1:[[0,0,1,0]] ,......] Y2:[[1,1,1,1],....] , [sx*tx, sy*ty, sw*tw, sh*th],...] X2, Y1, Y2, IouS = calc_iou(R, img_data, class_mapping ) if X2 is None: rpn_accuracy_rpn_monitor.append(0) rpn_accuracy_for_epoch.append(0) continue neg_samples = np.where(Y1[0, :, -1] == 1) #非背景 #直接对应坐标的位置 pos_samples = np.where(Y1[0, :, -1] == 0) #背景 if len(neg_samples) > 0: neg_samples = neg_samples[0] else: neg_samples = [] if len(pos_samples) > 0:
def main(args): vis = args.vis debug = args.debug save = args.save nparticles = args.particles root_path = '/home/ccjiang/Documents/caffe-fast-rcnn/examples/tracker/' dataset_path = "/data/OTB100" sequence = args.sequence model_def = os.path.join(root_path, args.prototxt) model_weight = os.path.join(root_path, args.caffemodel) for t in os.walk(os.path.join(dataset_path, sequence, sequence, "img")): if t[0] == os.path.join(dataset_path, sequence, sequence, "img"): nFrame = len(t[2]) print "Total frames are: ", nFrame gt_path = os.path.join(dataset_path, sequence, sequence, "groundtruth_rect.txt") gt_boxes = utils.get_boxes_all(gt_path) vggnet = VGGnet.VGGnet(model_def, model_weight) thre_min_neg = 0.0 thre_max_neg = 0.4 #0.5 thre_min_pos = 0.8 thre_max_pos = 1.0 conf_hist = [] iou_hist = [] area_hist = [] eig_hist = [] pred_hist = [] #(x1,y1,x2,y2) reinit = 0 nFrame = np.minimum(nFrame, gt_boxes.shape[0]) for id in np.arange(0, nFrame): #nFrame frame_name = "img/%04d.jpg" % (id + 1) print "Start processing: %s" % frame_name frame_path = os.path.join(dataset_path, sequence, sequence, frame_name) frame_data = caffe.io.load_image(frame_path) # (432,576,3), in [0,1] gt_box = gt_boxes[id] if id == 0: h, w, c = frame_data.shape frame_shape = [c, w, h] fps = 20 fourcc = cv2.VideoWriter_fourcc(*'MJPG') video_writer = cv2.VideoWriter("res_%s.avi" % sequence, fourcc, fps, (w, h)) fail_times = 0 area = (gt_box[2] - gt_box[0]) * (gt_box[3] - gt_box[1]) ratio = (gt_box[2] - gt_box[0]) / (gt_box[3] - gt_box[1] ) #ratio=w/h # set up net.blobs['im_info'] print "Image Size: ", w, h vggnet.reshape(w=w, h=h, nbox=nparticles) filter = PFfilter.PFfilter( utils.bbox_to_states(gt_box, area, ratio), area, ratio, w, h, nparticles) filter.create_particles() filter.restrict_particles(w, h) area_hist.append(filter.cur_a) pred_hist.append(np.array(gt_box).reshape(1, -1)) #pca # test sample_iou num_true = 500 num_false = 1000 boxes_train = [] #boxes_train_neg=[] iou_train = [] try: #Q=[[1,0],[0,1]] #for pixel wise Q = 0.05 #box_w,box_h sample_box_true, sample_iou_true = filter.sample_iou( gt_box, Q, 0.01, 0.01, num_true, 0.8, 1.0) except OverflowError as e: print "too many loops in sample." # print sample_box_true[:10] # print sample_box_true.shape[0] # print sample_iou_true[:10] print "average iou: ", np.mean(sample_iou_true) boxes_train.append(sample_box_true) iou_train.append(sample_iou_true) try: #Q=[[36,0],[0,36]]#for pixel wise Q = 0.2 #0.15 sample_box_false, sample_iou_false = filter.sample_iou( gt_box, Q, 0.2, 0.01, num_false / 2, 0, thre_max_neg) except OverflowError as e: print "too many loops in sample." # print sample_box_false[:10] # print sample_box_false.shape[0] # print sample_iou_false[:10] print "average iou: ", np.mean(sample_iou_false) boxes_train.append(sample_box_false) iou_train.append(sample_iou_false) try: #Q=[[36,0],[0,36]]#for pixel wise Q = 0.2 sample_box_false, sample_iou_false = filter.sample_iou( gt_box, Q, 0.01, 0.2, num_false / 2, 0, thre_max_neg) except OverflowError as e: print "too many loops in sample." # print sample_box_false[:10] # print sample_box_false.shape[0] # print sample_iou_false[:10] print "average iou: ", np.mean(sample_iou_false) boxes_train.append(sample_box_false) iou_train.append(sample_iou_false) boxes_train = np.vstack(boxes_train) iou_train = np.vstack(iou_train) y_train_true = np.ones((num_true, )) y_train_false = np.zeros((num_false, )) y_train = np.hstack([y_train_true, y_train_false]) #permutation ind_perm = np.random.permutation(range(num_false + num_true)) boxes_train = boxes_train[ind_perm, :] iou_train = iou_train[ind_perm] y_train = y_train[ind_perm] ind_pos = np.where(y_train == 1)[0] ind_neg = np.where(y_train == 0)[0] vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0]) features = vggnet.get_features_first_raw(frame_data, boxes_raw=boxes_train, id=id) for k, v in features.iteritems(): print k, v.shape if k == 'f3': pca3 = utils.skl_pca(v) v_pca3 = pca3.transform(v) pca3_pos = np.zeros((num_true, pca3.n_components_), dtype=np.float32) pca3_neg = np.zeros((num_false, pca3.n_components_), dtype=np.float32) pca3_pos[...] = v_pca3[ind_pos, :] pca3_neg[...] = v_pca3[ind_neg, :] #utils.vis_as_image(v_pca3) #plt.imshow(v_pca3) #plt.title("PCA features") #plt.show() #plt.close() #logistic regression y_weight = sklearn.utils.class_weight.compute_class_weight( class_weight='balanced', classes=np.array([0, 1]), y=y_train) #print y_weight class_weight = {0: y_weight[0], 1: y_weight[1]} clf3 = SVC(kernel="linear") #clf3=linear_model.LogisticRegression(fit_intercept=True,solver='liblinear') clf3.fit(v_pca3, y_train) score3 = clf3.score(v_pca3, y_train) print 'score3: ', score3 #prob=clf3.predict_proba(v_pca3) print clf3.classes_ #print prob vis_feature = False if vis_feature: utils.vis_features(features, id) start_time = time.time() else: if fail_times >= 5: #reinitialize reinit += 1 area = (gt_box[2] - gt_box[0]) * (gt_box[3] - gt_box[1]) ratio = (gt_box[2] - gt_box[0]) / (gt_box[3] - gt_box[1]) filter = PFfilter.PFfilter( utils.bbox_to_states(gt_box, area, ratio), area, ratio, w, h, nparticles) #filter.reset(utils.bbox_to_states(gt_box, area, ratio), area, ratio) filter.create_particles() filter.restrict_particles(w, h) area_hist.append(filter.cur_a) pred_box = gt_box boxes_train = [] pred_hist.append(np.array(gt_box).reshape(1, -1)) #pred_hist.append(pred_box) conf_hist.append(-0.1) # boxes_train_neg=[] iou_train = [] try: # Q=[[1,0],[0,1]] #for pixel wise Q = 0.05 # box_w,box_h sample_box_true, sample_iou_true = filter.sample_iou( gt_box, Q, 0.01, 0.01, num_true, 0.8, 1.0) except OverflowError as e: print "too many loops in sample." # print sample_box_true[:10] # print sample_box_true.shape[0] # print sample_iou_true[:10] print "average iou: ", np.mean(sample_iou_true) boxes_train.append(sample_box_true) iou_train.append(sample_iou_true) try: # Q=[[36,0],[0,36]]#for pixel wise Q = 0.2 #0.15 sample_box_false, sample_iou_false = filter.sample_iou( gt_box, Q, 0.2, 0.01, num_false / 2, 0, thre_max_neg) except OverflowError as e: print "too many loops in sample." # print sample_box_false[:10] # print sample_box_false.shape[0] # print sample_iou_false[:10] print "average iou: ", np.mean(sample_iou_false) boxes_train.append(sample_box_false) iou_train.append(sample_iou_false) try: # Q=[[36,0],[0,36]]#for pixel wise Q = 0.2 sample_box_false, sample_iou_false = filter.sample_iou( gt_box, Q, 0.01, 0.2, num_false / 2, 0, thre_max_neg) except OverflowError as e: print "too many loops in sample." # print sample_box_false[:10] # print sample_box_false.shape[0] # print sample_iou_false[:10] print "average iou: ", np.mean(sample_iou_false) boxes_train.append(sample_box_false) iou_train.append(sample_iou_false) boxes_train = np.vstack(boxes_train) iou_train = np.vstack(iou_train) y_train_true = np.ones((num_true, )) y_train_false = np.zeros((num_false, )) y_train = np.hstack([y_train_true, y_train_false]) # permutation ind_perm = np.random.permutation(range(num_false + num_true)) boxes_train = boxes_train[ind_perm, :] iou_train = iou_train[ind_perm] y_train = y_train[ind_perm] ind_pos = np.where(y_train == 1)[0] ind_neg = np.where(y_train == 0)[0] vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0]) features = vggnet.get_features_first_raw(frame_data, boxes_raw=boxes_train, id=id) for k, v in features.iteritems(): print k, v.shape if k == 'f3': v_pca3 = pca3.transform(v) pca3_pos[...] = v_pca3[ind_pos, :] pca3_neg[...] = v_pca3[ind_neg, :] clf3.fit(v_pca3, y_train) score3 = clf3.score(v_pca3, y_train) print 'score3: ', score3 # prob=clf3.predict_proba(v_pca3) print clf3.classes_ fail_times = 0 continue filter.predict_particles(Q=0.02, cr=0.05, ca=0.05) #0.02,0.0005,0.005 filter.restrict_particles(w, h) area_hist.append(filter.cur_a) #compute conf conf = np.zeros(filter.weights.shape) #np.save('particles.npy',filter.particles) pred_boxes = utils.state_to_bbox(filter.particles, area, ratio) vggnet.reshape(w, h, filter.num_particles) features = vggnet.get_features_first_raw(frame_data, boxes_raw=pred_boxes, id=id) for k, v in features.iteritems(): print k, v.shape if k == 'f3': v_pca3 = pca3.transform(v) #utils.vis_as_image(v_pca3) #plt.imshow(v_pca3) #plt.title("PCA features") #plt.show() #plt.close() #logistic regression #conf=clf3.predict_proba(v_pca3)[:,1] conf = -clf3.decision_function(v_pca3) conf_max = np.max(conf) conf_min = np.min(conf) print 'conf_max: ', conf_max print 'conf_min: ', conf_min filter.update_particles(conf) # pred_state = filter.estimate() print filter.weights filter.resample() # always resample pred_state, s_particles, r_particles = filter.estimate(k=10) cov_particles = np.dot( filter.particles[:, :4].T, filter.particles[:, :4]) / filter.particles.shape[0] eigval, eigvec = np.linalg.eig(cov_particles) max_val = eigval[0] eig_hist.append(max_val) print 'Max eigvalue: %f' % max_val #print 'conf is: ',conf if conf_max > 0 and max_val < 200000: fail_times = 0 else: fail_times += 1 #filter.update_particles(conf) #pred_state=filter.estimate() #filter.resample() #pred_state, s_particles, r_particles = filter.estimate(k=10) print "conf_max too low, not update particles " pred_box = utils.state_to_bbox(pred_state.reshape((-1, 6)), area, ratio) print 'ground truth bbox is: ', gt_box print "pred_box is: ", pred_box show_sr = False if show_sr: plt.hist2d(s_particles, r_particles, bins=50, weights=filter.weights.squeeze()) ''' plt.scatter(s_particles,r_particles,c='r',marker='.',linewidths=1) plt.xlabel('Area') plt.ylabel('Aspect ratio') plt.title('Area and Ratio of particles') plt.axis('equal') ''' plt.show() iou = utils.calc_iou(gt_box, pred_box) print 'iou is: ', iou pred_hist.append(pred_box) conf_hist.append(conf_max) iou_hist.append(iou) if conf_max >= 0.1: #0.5 #update pca3_pos and pca3_neg new_true = 100 #50 new_false = 200 #100 boxes_train = [] iou_train = [] Q = 0.02 try: sample_box_true, sample_iou_true = filter.sample_iou( pred_box, Q, 0.01, 0.01, new_true, 0.85, 1.0) except OverflowError as e: print "too many loops in sample." # print sample_box_true[:10] # print sample_box_true.shape[0] # print sample_iou_true[:10] print "average iou: ", np.mean(sample_iou_true) boxes_train.append(sample_box_true) iou_train.append(sample_iou_true) try: Q = 0.2 sample_box_false, sample_iou_false = filter.sample_iou( pred_box, Q, 0.2, 0.01, new_false / 2, 0, thre_max_neg) except OverflowError as e: print "too many loops in sample." # print sample_box_false[:10] # print sample_box_false.shape[0] # print sample_iou_false[:10] print "average iou: ", np.mean(sample_iou_false) boxes_train.append(sample_box_false) iou_train.append(sample_iou_false) try: Q = 0.2 sample_box_false, sample_iou_false = filter.sample_iou( pred_box, Q, 0.01, 0.2, new_false / 2, 0, thre_max_neg) except OverflowError as e: print "too many loops in sample." # print sample_box_false[:10] # print sample_box_false.shape[0] # print sample_iou_false[:10] print "average iou: ", np.mean(sample_iou_false) boxes_train.append(sample_box_false) iou_train.append(sample_iou_false) boxes_train = np.vstack(boxes_train) iou_train = np.vstack(iou_train) y_train_true = np.ones((new_true, )) y_train_false = np.zeros((new_false, )) y_train = np.hstack([y_train_true, y_train_false]) # permutation ind_perm = np.random.permutation(range(new_false + new_true)) boxes_train = boxes_train[ind_perm, :] y_train = y_train[ind_perm] new_y = np.zeros(y_train.shape) new_y[...] = y_train ind_pos = np.where(y_train == 1)[0] ind_neg = np.where(y_train == 0)[0] vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0]) features = vggnet.get_features_first_raw(frame_data, boxes_raw=boxes_train, id=id) for k, v in features.iteritems(): print k, v.shape if k == 'f3': v_pca3 = pca3.transform(v) #random substitude pca3_cur_pos = v_pca3[ind_pos, :] pca3_cur_neg = v_pca3[ind_neg, :] to_subst = random.sample(range(num_true), new_true) pca3_pos[to_subst, :] = pca3_cur_pos to_subst = random.sample(range(num_false), new_false) pca3_neg[to_subst, :] = pca3_cur_neg if conf_max < 0.1 and fail_times >= 2: #0.99 #if conf_max<0.95 and conf_max>0.5: #update classification model print 'updating model...' pca3_train = np.vstack([pca3_pos, pca3_neg]) y_train_true = np.ones((num_true, )) y_train_false = np.zeros((num_false, )) y_train = np.hstack([y_train_true, y_train_false]) # permutation ind_perm = np.random.permutation(range(num_false + num_true)) pca3_train = pca3_train[ind_perm, :] y_train = y_train[ind_perm] #logistic regression clf3.fit(pca3_train, y_train) print 'score is: ', clf3.score(pca3_train, y_train) #fail_times=0 # (B,G,R) frame_data_cv = frame_data * 255 # [0,1]-->[0,255] frame_data_cv = frame_data_cv[:, :, ::-1] # RGB->BGR frame_data_cv = frame_data_cv.astype('uint8') #cv2.rectangle(frame_data_cv, (int(gt_box[0]), int(gt_box[1])), (int(gt_box[2]), int(gt_box[3])), # (255, 0, 0), 2, 1) if id > 0: cv2.rectangle(frame_data_cv, (int(pred_box[0, 0]), int(pred_box[0, 1])), (int(pred_box[0, 2]), int(pred_box[0, 3])), (0, 255, 0), 2, 1) show_particles = False if show_particles: for i in range(filter.num_particles): cx = filter.particles[i, 0] cy = filter.particles[i, 1] cv2.circle(frame_data_cv, (int(cx), int(cy)), 1, (0, 0, 255), thickness=1) show_box = False if show_box: n = 0 for i in ind_pos: if n % 5 == 0: cv2.rectangle( frame_data_cv, (int(boxes_train[i, 0]), int(boxes_train[i, 1])), (int(boxes_train[i, 2]), int(boxes_train[i, 3])), (0, 0, 255), 2, 1) n += 1 n = 0 show_particles_init = False if show_particles_init: for i in range(filter.num_particles): cx = filter.particles[i, 0] cy = filter.particles[i, 1] cv2.circle(frame_data_cv, (int(cx), int(cy)), 1, (0, 255, 0), thickness=1) show_frame = False #cv2.circle(frame_data_cv, (int(filter.cur_c[0]), int(filter.cur_c[1])), 2, (0, 0, 255), thickness=1) if show_frame: cv2.imshow(sequence, frame_data_cv) c = cv2.waitKey(1) if c != -1: if chr(c) == 'p': c = cv2.waitKey() #print 'You press: ',chr(c) #if chr(c)=='c': if chr(c) == 'c': cv2.destroyWindow(sequence) #conf_hist=np.array(conf_hist) #iou_hist=np.array(iou_hist) #np.save('conf_hist.npy',conf_hist) #np.save('iou_hist.npy',iou_hist) break else: video_writer.write(frame_data_cv) end_time = time.time() video_writer.release() iou_hist = np.array(iou_hist) pred_hist = np.array(pred_hist).squeeze() print "iou_hist: ", iou_hist.shape print "pred_hist: ", pred_hist.shape print "get_boxes: ", gt_boxes.shape precisions, auc_pre = utils.calc_prec(gt_boxes, pred_hist) print "precision is: %f" % (precisions[19]) suc, auc_iou = utils.calc_success(iou_hist) print "Average IOU is: %f" % (np.mean(iou_hist)) print "Auc of precision is: %f" % (auc_pre) print "Auc of success is: %f" % auc_iou print "Reinit times: %d" % reinit print "Average FPS: %f" % ((id + 1) / (end_time - start_time))
gt = (gt * 255).long().squeeze_(dim=1) image = image.to(device) gt = gt.to(device) salient_map = salient_map.to(device) # print(torch.unique(gt), gt.shape) x = torch.cat((image, salient_map), dim=1) optimizer.zero_grad() y, pbr_y = model(x) # loss = loss_func(y, gt) loss = loss_func(y, gt) + loss_func(pbr_y, gt) loss.backward() optimizer.step() y_label = softmax_to_label(y) pbr_y_label = softmax_to_label(pbr_y) y_iou = calc_iou(y_label, gt) pbr_y_iou = calc_iou(pbr_y_label, gt) losses.append(loss.item()) pbr_y_ious.append(pbr_y_iou.item()) y_ious.append(y_iou.item()) print( f"train: epoch {i+1}, step {index + 1}, loss: {loss}, iou: {y_iou}, pbr iou: {pbr_y_iou}." ) # break scheduler.step() # print(optimizer.state_dict()['param_groups'][0]['lr']) # learning rate torch.save(model.state_dict(), f"./models/checkpoints.pth") print( f"epoch {i+1} training end, mean loss: {np.mean(losses)}, iou: {np.mean(y_ious)}, pbr iou: {np.mean(pbr_y_ious)}." )
def main(args): vis = args.vis debug = args.debug save = args.save nparticles = args.particles root_path = '/home/ccjiang/Documents/py-faster-rcnn/caffe-fast-rcnn/examples/tracker/' dataset_path = args.dataset # "/data/OTB100" dataset100_seq = ['Bird2', 'BlurCar1', 'BlurCar3', 'BlurCar4', 'Board', 'Bolt2', 'Boy', 'Car2', 'Car24', 'Coke', 'Coupon', 'Crossing', 'Dancer', 'Dancer2', 'David2', 'David3', 'Dog', 'Dog1', 'Doll', 'FaceOcc1', 'FaceOcc2', 'Fish', 'FleetFace', 'Football1', 'Freeman1', 'Freeman3', 'Girl2', 'Gym', 'Human2', 'Human5', 'Human7', 'Human8', 'Jogging', 'KiteSurf', 'Lemming', 'Man', 'Mhyang', 'MountainBike', 'Rubik', 'Singer1', 'Skater', 'Skater2', 'Subway', 'Suv', 'Tiger1', 'Toy', 'Trans', 'Twinnings', 'Vase'] dataset50_seq = ['Basketball', 'Bird1', 'BlurBody', 'BlurCar2', 'BlurFace', 'BlurOwl', 'Bolt', 'Box', 'Car1', 'Car4', 'CarDark', 'CarScale', 'ClifBar', 'Couple', 'Crowds','David', 'Deer', 'Diving', 'DragonBaby', 'Dudek', 'Football', 'Freeman4', 'Girl', 'Human3', 'Human4', 'Human6', 'Human9', 'Ironman', 'Jump', 'Jumping', 'Liquor', 'Matrix', 'MotorRolling', 'Panda', 'RedTeam', 'Shaking', 'Singer2', 'Skating1', 'Skating2', 'Skiing', 'Soccer', 'Surfer', 'Sylvester', 'Tiger2', 'Trellis', 'Walking', 'Walking2', 'Woman'] datafull_seq = dataset100_seq + dataset50_seq if "OTB50" in dataset_path: data_seq = dataset50_seq else: data_seq = dataset100_seq log_name = 'log_1119.txt' log_file = open(log_name, 'w') records_success = [] # defaultdict(list) records_precision = [] # defaultdict(list) records_reinit = defaultdict(list) model_def = os.path.join(root_path, args.prototxt) model_weight = os.path.join(root_path, args.caffemodel) vggnet = VGGnet.VGGnet(model_def, model_weight) thre_max_neg = 0.3 # 0.5 test_times = 1 # 0 for t in range(test_times): print 'Test round: %d' % t log_file.write('Test round: %d\n' % t) # sequences = ['Fish'] for sequence in datafull_seq: # datafull_seq if sequence in dataset50_seq: dataset_path = "/data/OTB50" else: dataset_path = "/data/OTB100" for t in os.walk(os.path.join(dataset_path, sequence, sequence, "img")): if t[0] == os.path.join(dataset_path, sequence, sequence, "img"): nFrame = len(t[2]) print 'Processing: %s' % sequence log_file.write('Processing: %s\n' % sequence) print "Total frames are: ", nFrame log_file.write('Total frames are: %d\n' % nFrame) gt_path = os.path.join(dataset_path, sequence, sequence, "groundtruth_rect.txt") gt_boxes = utils.get_boxes_all(gt_path) conf_hist = [] iou_hist = [] area_hist = [] pred_hist = [] # (x1,y1,x2,y2) eig_hist = [] reinit = 0 nFrame = np.minimum(nFrame, gt_boxes.shape[0]) id_shift = 0 init_id = False update_recent = False for id in np.arange(0, nFrame): frame_name = "img/%04d.jpg" % (id + 1) # print "Start processing: %s" % frame_name frame_path = os.path.join(dataset_path, sequence, sequence, frame_name) if os.path.exists(frame_path) == False: id_shift = id_shift + 1 continue id = id - id_shift frame_data = caffe.io.load_image(frame_path) # (432,576,3), in [0,1] gt_box = gt_boxes[id] if init_id == False: h, w, c = frame_data.shape frame_shape = [c, w, h] fps = 20 fourcc = cv2.VideoWriter_fourcc(*'MJPG') video_writer = cv2.VideoWriter("res_%s.avi"%sequence,fourcc,fps,(w,h)) fail_times = 0 box_w = gt_box[2] - gt_box[0] box_h = gt_box[3] - gt_box[1] area = (gt_box[2] - gt_box[0]) * (gt_box[3] - gt_box[1]) ratio = (gt_box[2] - gt_box[0]) / (gt_box[3] - gt_box[1]) # ratio=w/h # set up net.blobs['im_info'] print "Image Size: ", w, h log_file.write('Image Size: %d %d\n' % (w, h)) b = gt_box[np.newaxis, :] vggnet.reshape(w=w, h=h, nbox=b.shape[0]) features0 = vggnet.get_features("conv3_3", frame_data, boxes_raw=b) # shape:(256,hs,ws),conv3_3,res3b3 features0 = np.squeeze(features0) pca_f, scaler_f = featmap_pca2(features0,ncompnents=128)#128 box_w = gt_box[2] - gt_box[0] box_h = gt_box[3] - gt_box[1] vggnet.reshape(w=w, h=h, nbox=nparticles) pfilter = PFfilter.PFfilter(utils.bbox_to_states(gt_box, area, ratio), area, ratio, w, h, nparticles) pfilter.create_particles() pfilter.restrict_particles(w, h) area_hist.append(pfilter.cur_a) pred_hist.append(np.array(gt_box).reshape(1, -1)) # pca # test sample_iou num_true = 500 num_false = 1000 # 1000 #data augument gt_box_otb = gt_box.copy() gt_box_otb[2] -= gt_box_otb[0] gt_box_otb[3] -= gt_box_otb[1] boxes_train = [] ids = np.zeros(num_false + num_true) imgs = [] for i in np.arange(4): if i == 0: img1,gt1,img2,gt2,img3,gt3 = DataAugment(frame_data,gt_box_otb,True) gt1[2] += gt1[0] gt1[3] += gt1[1] gt2[2] += gt2[0] gt2[3] += gt2[1] gt3[2] += gt3[0] gt3[3] += gt3[1] box_true1, iou_true = pfilter.sample_iou_pred_box(gt1, 0.05, 0.01, 0.01, 20, 0.8, 1.0) box_true2, iou_true = pfilter.sample_iou_pred_box(gt2, 0.05, 0.01, 0.01, 40, 0.8, 1.0) box_true3, iou_true = pfilter.sample_iou_pred_box(gt3, 0.05, 0.01, 0.01, 20, 0.8, 1.0) box_true1[0, ...] = gt1 box_true2[0, ...] = gt2 box_true3[0, ...] = gt3 boxes_train.append(box_true1) boxes_train.append(box_true2) boxes_train.append(box_true3) imgs.append(img1) imgs.append(img2) imgs.append(img3) ids[20:60] = 1 ids[60:80] = 2 else: img1, gt1, img2, gt2 = DataAugment(frame_data, gt_box_otb,False) gt1[2] += gt1[0] gt1[3] += gt1[1] gt2[2] += gt2[0] gt2[3] += gt2[1] box_true1, iou_true = pfilter.sample_iou_pred_box(gt1, 0.05, 0.01, 0.01, 20, 0.8, 1.0) box_true2, iou_true = pfilter.sample_iou_pred_box(gt2, 0.05, 0.01, 0.01, 20, 0.8, 1.0) box_true1[0, ...] = gt1 box_true2[0, ...] = gt2 boxes_train.append(box_true1) boxes_train.append(box_true2) imgs.append(img1) imgs.append(img2) cur_i = 80+(i-1)*40 ids[cur_i:(cur_i+20)] = 3+(i-1)*2 ids[(cur_i+20):(cur_i+40)] = 3+(i-1)*2+1 # boxes_train_neg=[] try: # Q=[[1,0],[0,1]] #for pixel wise Q = 0.05 # box_w,box_h,0.05 sample_box_true, sample_iou_true = pfilter.sample_iou_pred_box(gt_box, Q, 0.01, 0.01, num_true-200, 0.8, 1.0)#0.8 except OverflowError as e: print "too many loops in sample in Initialize--TRUE." boxes_train.append(sample_box_true) try: # Q=[[36,0],[0,36]]#for pixel wise Q = 0.2 # 0.2 sample_box_false, sample_iou_false = pfilter.sample_iou(gt_box, Q, 0.2, 0.01, num_false / 2, 0, thre_max_neg) # 0.2,0.01 except OverflowError as e: print "too many loops in sample in Initialize--FALSE." # print sample_box_false[:10] # print sample_box_false.shape[0] # print sample_iou_false[:10] # print "average iou: ", np.mean(sample_iou_false) boxes_train.append(sample_box_false) try: # Q=[[36,0],[0,36]]#for pixel wise Q = 0.2 # 0.2 sample_box_false, sample_iou_false = pfilter.sample_iou(gt_box, Q, 0.01, 0.2, num_false / 2, 0, thre_max_neg) # 0.01,0.2 except OverflowError as e: print "too many loops in sample in Initialize--FALSE." boxes_train.append(sample_box_false) boxes_train = np.vstack(boxes_train) imgs.append(frame_data) imgs = np.stack(imgs,axis=0)#(10,h,w,c) ids[200:] = 9 y_train_true = np.ones((num_true,)) y_train_false = np.zeros((num_false,)) ids_save = np.ones((num_true+num_false)) ids_save[num_true:] = 0 ids_save[20:60] = 2 y_train = np.hstack([y_train_true, y_train_false]) # permutation ind_perm = np.random.permutation(range(num_false + num_true)) boxes_train = boxes_train[ind_perm, :] ids_save = ids_save[ind_perm] y_train = y_train[ind_perm] ids = ids[ind_perm] ind_pos = np.where(y_train == 1)[0] ind_neg = np.where(y_train == 0)[0] vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0],batch_size=10) #features = vggnet.get_features_first_raw(frame_data, boxes_raw=boxes_train, id=id) features = vggnet.get_features_first_id(imgs,boxes_raw=boxes_train,id=ids) #features = vggnet.get_features_first_sel(frame_data, boxes_raw=boxes_train, id=id, sel=f_inds) for k, v in features.iteritems(): # print k,v.shape if k == 'f3': #pca3, scaler1, nPCA = utils.skl_pca2(v) v = feat_transformpca(pca_f, scaler_f, v) # (N,128,7,7) #pca3,scaler,nPCA=utils.skl_pca2(v) #v_pca3 = pca3.transform(scaler.transform(v)) #np.save("pca_results/testpca_%s.npy"%sequence,v_pca3) #np.save('labelpca.npy',y_train) #np.save("pca_results/label_pca_%s"%sequence,ids_save) pca3_pos = np.zeros((num_true, pca_f.n_components_*49), dtype=np.float32) pca3_neg = np.zeros((num_false, pca_f.n_components_*49), dtype=np.float32) pca3_pos[...] = v[ind_pos, :] pca3_neg[...] = v[ind_neg, :] # utils.vis_as_image(v_pca3) # plt.imshow(v_pca3) # plt.title("PCA features") # plt.show() # plt.close() # logistic regression y_weight = sklearn.utils.class_weight.compute_class_weight(class_weight='balanced', classes=np.array([0, 1]), y=y_train) # print y_weight class_weight = {0: y_weight[0], 1: y_weight[1]} clf3 = linear_model.LogisticRegression(fit_intercept=True, solver='liblinear') clf3.fit(v, y_train) vis_feature = False if vis_feature: utils.vis_features(features, id) start_time = time.time() else: if fail_times >= 5: # reinitialize update_recent = False reinit += 1 area = (gt_box[2] - gt_box[0]) * (gt_box[3] - gt_box[1]) ratio = (gt_box[2] - gt_box[0]) / (gt_box[3] - gt_box[1]) pfilter = PFfilter.PFfilter(utils.bbox_to_states(gt_box, area, ratio), area, ratio, w, h, nparticles) # filter.reset(utils.bbox_to_states(gt_box, area, ratio), area, ratio) pfilter.create_particles() pfilter.restrict_particles(w, h) area_hist.append(pfilter.cur_a) pred_box = gt_box pred_hist.append(np.array(pred_box).reshape(1, -1)) conf_hist.append(-0.1) boxes_train = [] # boxes_train_neg=[] iou_train = [] try: # Q=[[1,0],[0,1]] #for pixel wise Q = 0.05 # box_w,box_h,0.05 sample_box_true, sample_iou_true = pfilter.sample_iou_pred_box(gt_box, Q, 0.01, 0.01, num_true, 0.8, 1.0) except OverflowError as e: print "too many loops in sample in Reinitialize--TRUE." boxes_train.append(sample_box_true) iou_train.append(sample_iou_true) try: # Q=[[36,0],[0,36]]#for pixel wise Q = 0.2 # 0.2 sample_box_false, sample_iou_false = pfilter.sample_iou(gt_box, Q, 0.01, 0.2, num_false / 2, 0, thre_max_neg) except OverflowError as e: print "too many loops in sample in Reinitialize--FALSE." boxes_train.append(sample_box_false) iou_train.append(sample_iou_false) try: # Q=[[36,0],[0,36]]#for pixel wise Q = 0.2 # 0.2 sample_box_false, sample_iou_false = pfilter.sample_iou(gt_box, Q, 0.2, 0.01, num_false / 2, 0, thre_max_neg) except OverflowError as e: print "too many loops in sample in Reinitialize--FALSE." # print sample_box_false[:10] # print sample_box_false.shape[0] # print sample_iou_false[:10] # print "average iou: ", np.mean(sample_iou_false) boxes_train.append(sample_box_false) iou_train.append(sample_iou_false) boxes_train = np.vstack(boxes_train) iou_train = np.vstack(iou_train) y_train_true = np.ones((num_true,)) y_train_false = np.zeros((num_false,)) y_train = np.hstack([y_train_true, y_train_false]) # permutation ind_perm = np.random.permutation(range(num_false + num_true)) boxes_train = boxes_train[ind_perm, :] iou_train = iou_train[ind_perm] y_train = y_train[ind_perm] ind_pos = np.where(y_train == 1)[0] ind_neg = np.where(y_train == 0)[0] vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0]) features = vggnet.get_features_first_raw(frame_data, boxes_raw=boxes_train, id=id) #features=feat_transformpca(pca_f,scaler_f,features) #features = vggnet.get_features_first_sel(frame_data, boxes_raw=boxes_train, id=id, sel=f_inds) for k, v in features.iteritems(): # print k, v.shape if k == 'f3': v = feat_transformpca(pca_f, scaler_f, v) # (N,128,7,7) #v_pca3 = pca3.transform(scaler.transform(v)) pca3_pos[...] = v[ind_pos, :] pca3_neg[...] = v[ind_neg, :] clf3 = linear_model.LogisticRegression(fit_intercept=True, solver='liblinear') clf3.fit(v, y_train) # score3 = clf3.score(v_pca3, y_train) # print 'score3: ', score3 # prob=clf3.predict_proba(v_pca3) # print clf3.classes_ fail_times = 0 continue pfilter.predict_particles(Q=0.2, cr=0.01, ca=0.01) # 0.2,0.01 pfilter.restrict_particles(w, h) area_hist.append(pfilter.cur_a) # compute conf # conf = np.zeros(pfilter.weights.shape) # np.save('particles.npy',filter.particles) pred_boxes = utils.state_to_bbox(pfilter.particles, area, ratio) #add Gaussian regularization if id>1: gauss_sig = 0.5 gauss_w = np.exp(-np.square((pfilter.particles[:,0]-pred_state[0])/(gauss_sig*box_w)/2.0)-np.square((pfilter.particles[:,1]-pred_state[1])/(gauss_sig*box_h)/2.0)) pfilter.update_particles(gauss_w) print gauss_w vggnet.reshape(w, h, pfilter.num_particles) features = vggnet.get_features_first_raw(frame_data, boxes_raw=pred_boxes, id=id) #features=feat_transformpca(pca_f,scaler_f,features) #features = vggnet.get_features_first_sel(frame_data, boxes_raw=pred_boxes, id=id, sel=f_inds) for k, v in features.iteritems(): # print k,v.shape if k == 'f3': v = feat_transformpca(pca_f, scaler_f, v) # (N,128,7,7) vf = v #v_pca3 = pca3.transform(scaler.transform(v)) conf = clf3.predict_proba(v)[:, 1] # process preds to find out pred_box in terms of confm conf = np.array(conf) conf_max = np.max(conf) conf_min = np.min(conf) pfilter.update_particles(conf) # do resample first or estimate first? # filter.resample() # always resample pred_state, s_particles, r_particles = pfilter.estimate(k=10) pfilter.resample() pred_box = utils.state_to_bbox(pred_state.reshape((-1, 6)), area, ratio) hard,hard_negv = nms_pred(pred_box,pred_boxes,vf,conf) if hard: hard_negvN = hard_negv.shape[0] #print hard_negv.shape else: hard_negvN = 0 avg_pos = np.mean(pfilter.particles[:, :2], axis=0, keepdims=True) # avg_pos[:,0]/=w # avg_pos[:,1]/=h ptls_avg = (pfilter.particles[:, :2] - avg_pos) / np.array([[box_w, box_h]]) cov_particles = np.dot(ptls_avg.T, ptls_avg) / pfilter.particles.shape[ 0] eigval, eigvec = np.linalg.eig(cov_particles) max_val = eigval[0] eig_hist.append(max_val) print 'Max eigvalue: %f' % max_val # print 'conf is: ',conf if conf_max > 0.5: # 0.8 fail_times = 0 update_recent = False else: fail_times += 1 show_sr = False if show_sr: count, xedge, yedge, tmp_im = plt.hist2d(s_particles, r_particles, bins=10, weights=pfilter.weights.squeeze(), cmap=plt.cm.gray) top3 = np.argsort(-count, axis=None)[:3] row_ind = top3[:] / count.shape[1] col_ind = top3[:] % count.shape[0] plt.show() print pred_box iou = utils.calc_iou(gt_box, pred_box) # print 'iou is: ', iou pred_hist.append(pred_box) conf_hist.append(conf_max) iou_hist.append(iou) if conf_max >= 0.7: # 0.5 # update pca3_pos and pca3_neg new_true = 100 # 100 new_false = 400 # 200 boxes_train = [] iou_train = [] Q = 0.05 # 0.02 try: sample_box_true, sample_iou_true = pfilter.sample_iou_pred_box(pred_box, Q, 0.01, 0.01, new_true, 0.85, 1.0) except OverflowError as e: print "too many loops in sample in Update--TRUE." # print sample_box_true[:10] # print sample_box_true.shape[0] # print sample_iou_true[:10] # print "average iou: ", np.mean(sample_iou_true) boxes_train.append(sample_box_true) iou_train.append(sample_iou_true) # part_iou=utils.calc_iou(pred_box,pred_boxes) # ind_iou=np.where(part_iou<0.3)[0] # ind_n=np.minimum(new_false/2,ind_iou.shape[0]) # boxes_train.append(pred_boxes[ind_iou[:ind_n],:]) # iou_train.append(part_iou[ind_iou]) new_false_left = new_false - hard_negvN # -ind_n try: Q = 0.2 # 0.2 sample_box_false, sample_iou_false = pfilter.sample_iou_pred_box(pred_box, Q, 0.2, 0.01, (new_false_left + 1) / 2, 0, thre_max_neg) except OverflowError as e: print "too many loops in sample in Update--FALSE." # print sample_box_false[:10] # print sample_box_false.shape[0] # print sample_iou_false[:10] # print "average iou: ", np.mean(sample_iou_false) boxes_train.append(sample_box_false) iou_train.append(sample_iou_false) try: Q = 0.2 # 0.2 sample_box_false, sample_iou_false = pfilter.sample_iou_pred_box(pred_box, Q, 0.01, 0.2, new_false_left / 2, 0, thre_max_neg) except OverflowError as e: print "too many loops in sample in Update--FALSE." boxes_train.append(sample_box_false) iou_train.append(sample_iou_false) boxes_train = np.vstack(boxes_train) # iou_train = np.vstack(iou_train) vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0]) features = vggnet.get_features_second_raw(boxes_raw=boxes_train, id=id) #features = feat_transformpca(pca_f,scaler_f,features) #features = vggnet.get_features_second_sel(boxes_raw=boxes_train, id=id, sel=f_inds) for k, v in features.iteritems(): # print k, v.shape if k == 'f3': v = feat_transformpca(pca_f, scaler_f, v) # (N,128,7,7) #v_pca3 = pca3.transform(scaler.transform(v)) if hard: print v.shape print hard_negv.shape v = np.vstack([v,hard_negv]) y_train_true = np.ones((new_true,)) y_train_false = np.zeros((new_false,)) y_train = np.hstack([y_train_true, y_train_false]) # permutation ind_perm = np.random.permutation(range(new_false + new_true)) #boxes_train = boxes_train[ind_perm, :] v = v[ind_perm,:] y_train = y_train[ind_perm] new_y = np.zeros(y_train.shape) new_y[...] = y_train ind_pos = np.where(y_train == 1)[0] ind_neg = np.where(y_train == 0)[0] # random substitude pca3_cur_pos = v[ind_pos, :] pca3_cur_neg = v[ind_neg, :] to_subst = random.sample(range(num_true), new_true) pca3_pos[to_subst, :] = pca3_cur_pos to_subst = random.sample(range(num_false), new_false) pca3_neg[to_subst, :] = pca3_cur_neg if conf_max < 1 and fail_times >= 2 and update_recent==False: # if id%10==0: update_recent = True pca3_train = np.vstack([pca3_pos, pca3_neg]) y_train_true = np.ones((num_true,)) y_train_false = np.zeros((num_false,)) y_train = np.hstack([y_train_true, y_train_false]) # permutation ind_perm = np.random.permutation(range(num_false + num_true)) pca3_train = pca3_train[ind_perm, :] y_train = y_train[ind_perm] # logistic regression clf3 = linear_model.LogisticRegression(fit_intercept=True, solver='liblinear') clf3.fit(pca3_train, y_train) # print 'score is: ',clf3.score(pca3_train,y_train) # (B,G,R) frame_data_cv = frame_data * 255 # [0,1]-->[0,255] frame_data_cv = frame_data_cv[:, :, ::-1] # RGB->BGR frame_data_cv = frame_data_cv.astype('uint8') cv2.rectangle(frame_data_cv, (int(gt_box[0]), int(gt_box[1])), (int(gt_box[2]), int(gt_box[3])), (255, 0, 0), 2, 1) if id > 0 and init_id == True: cv2.rectangle(frame_data_cv, (int(pred_box[0, 0]), int(pred_box[0, 1])), (int(pred_box[0, 2]), int(pred_box[0, 3])), (0, 255, 0), 2, 1) if init_id == False: init_id = True show_particles = False if show_particles: for i in range(filter.num_particles): cx = pfilter.particles[i, 0] cy = pfilter.particles[i, 1] cv2.circle(frame_data_cv, (int(cx), int(cy)), 1, (0, 0, 255), thickness=1) show_box = False if show_box: n = 0 for i in ind_pos: if n % 5 == 0: cv2.rectangle(frame_data_cv, (int(boxes_train[i, 0]), int(boxes_train[i, 1])), (int(boxes_train[i, 2]), int(boxes_train[i, 3])), (0, 0, 255), 2, 1) n += 1 n = 0 show_particles_init = False if show_particles_init: for i in range(filter.num_particles): cx = pfilter.particles[i, 0] cy = pfilter.particles[i, 1] cv2.circle(frame_data_cv, (int(cx), int(cy)), 1, (0, 255, 0), thickness=1) show_frame = False cv2.circle(frame_data_cv, (int(pfilter.cur_c[0]), int(pfilter.cur_c[1])), 2, (0, 0, 255), thickness=1) if show_frame: cv2.imshow(sequence, frame_data_cv) c = cv2.waitKey(1) if c != -1: cv2.destroyWindow(sequence) break else: video_writer.write(frame_data_cv) end_time = time.time() video_writer.release() print "Average FPS: %f" % (nFrame / (end_time - start_time)) log_file.write("Average FPS: %f\n" % (nFrame / (end_time - start_time))) conf_hist = np.array(conf_hist) iou_hist = np.array(iou_hist) area_hist = np.array(area_hist) pred_hist = np.vstack(pred_hist) precisions, auc_pre = utils.calc_prec(gt_boxes, pred_hist) suc, auc_iou = utils.calc_success(iou_hist) records_precision.append(precisions * nFrame) records_success.append(suc * nFrame) print 'Precision @20 is: %f' % precisions[19] print 'Auc of Precision is: %f' % auc_pre print 'Auc of Success is: %f' % auc_iou print 'Reinit times: %d' % reinit log_file.write("Precision @20 is: %f\n" % precisions[19]) log_file.write('Auc of Precision is: %f\n' % auc_pre) log_file.write('Auc of Success is: %f\n' % auc_iou) log_file.write('Reinit times: %d\n' % reinit) #log_file.write('Selected feature maps: %d\n' % f_inds.shape[0]) #log_file.write('PCA components: %d\n' % nPCA) #res_f = open('results11/%s.txt'%sequence,'w') #pred_hist[:,2:] = pred_hist[:,2:] - pred_hist[:,:2] #res_f = write_res(pred_hist,res_f) #res_f.close() log_file.close() pkl = open('results_1031.pkl', 'w') pickle.dump([records_precision, records_success], pkl) pkl.close()
def _evaluate_worker(image_path, cfg, weights, output_names, input_size_wh, score_thresh, iou_thresh, labels, save_size_wh, save_dir_path): """ Need to add False Positive """ image_path = image_path.strip() image = cv2.imread(image_path) net = cv2.dnn.readNetFromDarknet(cfg, weights) idxs, boxes, scores, class_ids = \ infer.infer_image(image, net, output_names, input_size_wh, score_thresh) label_filename = os.path.splitext(image_path)[0] + '.txt' all_test_pass = True f_label = list(open(label_filename, 'r')) for label in f_label: label = label.strip().split(' ') t_class_id = int(label[0]) t_cbox = np.array(tuple(map(float, label[1:5]))) t_cbox = t_cbox * np.array([*input_size_wh, *input_size_wh]) t_bbox = utils.cbox2bbox(t_cbox) label_test_pass = False if len(idxs) >= 1: for i in idxs.flatten(): if t_class_id != class_ids[i]: continue cbox = boxes[i][:4] * np.array([*input_size_wh, *input_size_wh]) bbox = utils.cbox2bbox(cbox) iou = utils.calc_iou(bbox, t_bbox) if iou >= iou_thresh: label_test_pass = True break if not label_test_pass: all_test_pass = False break if not all_test_pass: save_image = cv2.resize(image, save_size_wh) save_image_true = save_image.copy() save_image_pred = save_image.copy() for label in f_label: label = label.strip().split(' ') t_class_id = int(label[0]) t_cbox = np.array(tuple(map(float, label[1:5]))) t_cbox = t_cbox * np.array([*save_size_wh, *save_size_wh]) t_x1, t_y1, t_x2, t_y2 = list(map(int, utils.cbox2bbox(t_cbox))) cv2.rectangle( save_image_true, (t_x1,t_y1), (t_x2,t_y2), (0,255,0), 2) text = f'{labels[t_class_id]}' cv2.putText( save_image_true, text, (t_x1+5, t_y2-15), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,255,0), 2) if len(idxs) >= 1: for i in idxs.flatten(): cbox = boxes[i][:4] * np.array([*save_size_wh, *save_size_wh]) x1, y1, x2, y2 = list(map(int, utils.cbox2bbox(cbox))) cv2.rectangle(save_image_pred, (x1,y1), (x2,y2), (0,255,0), 2) label = f'{labels[class_ids[i]]}' cv2.putText( save_image_pred, label, (x1+5, y2-15), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,255,0), 2) score = f'{scores[i]:.2f}' cv2.putText( save_image_pred, score, (x1+5, y2+15), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0,255,0), 1) stack_image = np.vstack((save_image, save_image_true, save_image_pred)) cv2.imwrite( os.path.join(save_dir_path, os.path.basename(image_path)), stack_image) return 1 return 0
def train(): torch.manual_seed(1280) torch.cuda.manual_seed(1280) np.random.seed(1280) # setup device device = torch.device("cuda" if torch.cuda.is_available() else "cpu") print("device = {}".format(device)) # load data data_loader_train = Cityscapes_Loader(path_root=path_root,split="train",n_classes=n_class) data_loader_val = Cityscapes_Loader(path_root=path_root,split="val",n_classes=n_class) train_loader = data.DataLoader(data_loader_train,batch_size=batch_size,num_workers=16) val_loader = data.DataLoader(data_loader_val,batch_size=batch_size,num_workers=16) # val batch_size=1 # set model assert torch.cuda.is_available(), "先把下面的cuda()删掉,debug阶段,不支持cpu" # if torch.cuda.is_available(): # torch.backends.cudnn.benchmark = True pretrain_model = VGGNet(requires_grad=True, remove_fc=True) model = FCNs(pretrained_net=pretrain_model, n_class=n_class) print("model loading success...") # set model running devices model.to(device) model = torch.nn.DataParallel(model, device_ids=range(torch.cuda.device_count())) print("usable gpu num: {}".format(torch.cuda.device_count())) # set optimizer, lr_scheduler, loss function optimizer = None if optimizer_name == "rmsprop": optimizer = torch.optim.RMSprop(model.parameters(),lr=learn_rate,momentum=momentum,weight_decay=weight_decay) elif optimizer_name == "sgd": optimizer = torch.optim.SGD(model.parameters(),lr=learn_rate,momentum=momentum,weight_decay=weight_decay) elif optimizer_name == "adam": optimizer = torch.optim.Adam(model.parameters(),lr=learn_rate) #criterion = mybasilLoss() criterion = torch.nn.BCEWithLogitsLoss() lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=step_size, gamma=gamma) # load checkpoints last_best_iou = -100. load_ckpt_path = os.path.join("checkpoints",load_ckpt_name) if is_load_checkpoints: if torch.cuda.is_available(): checkpoint = torch.load(load_ckpt_path) else: checkpoint = torch.load(load_ckpt_path, map_location='cpu') model.load_state_dict(checkpoint['model_state']) last_best_iou = checkpoint['best_iou'] start_epoch = checkpoint['epoch'] print('Checkpoint resume success... last iou:{:.4%}'.format(last_best_iou)) # train epoch best_iou = last_best_iou time_epoch = time.time() i = 0 for epoch in range(epochs): time_step = time.time() for step, batch in enumerate(train_loader): lr_scheduler.step(epoch=epoch) # 这个scheduler放在哪里?上一个for,还是这个for model.train() images = batch[0].to(device) #labels = batch[1].to(device) targets = batch[2].to(device) # targets.shape=[batch, n_classes, H, W] optimizer.zero_grad() outputs = model(images) loss = None try: loss = criterion(input=outputs,target=targets) except: torch.cuda.empty_cache() loss = criterion(input=outputs, target=targets) loss.backward() optimizer.step() if(step%100 == 0): print("train after setp:{}, Loss:{:.4%}, time used:{:.4} s".format(step,loss.item(),time.time()-time_step)) time_step = time.time() writer.add_scalars('ForTestOnly_record/Loss', {'train_loss': loss}, i + 1) i += 1 # each epoch save the checkpoint model.eval() with torch.no_grad(): total_iou = 0 for step_val, batch_val in enumerate(val_loader): images_val = batch_val[0].to(device) labels_val = batch_val[1].to(device) targets_val = batch_val[2].to(device) outputs_val = model(images_val) loss_val = criterion(input=outputs_val,target=targets_val) pred = outputs_val.data.max(1)[1].cpu().numpy() # 将mask格式[ gt = labels_val.data.cpu().numpy() #break # for only one val batch total_iou += calc_iou(pred,gt,n_class) mean_iou = total_iou/step_val print("epoch:{},loss_val:{:.4%},iou:{:.2%},total time used:{:.4}s".format(epoch + 1, loss_val, mean_iou,time.time()-time_epoch)) writer.add_scalars('Train_record/Loss',{'train_loss':loss,'val_loss':loss_val}, epoch+1) writer.add_scalars('Train_record/iou',{"iou":mean_iou}, epoch+1) time_epoch = time.time() if mean_iou >= best_iou: best_iou = mean_iou state ={ "epoch":epoch + 1, "model_state":model.state_dict(), "best_iou":best_iou, } if not os.path.isdir('./checkpoints'): os.mkdir('./checkpoints') save_path = os.path.join('./checkpoints', "eph_{}_iou_{:.2%}.ckpt.pth".format(epoch+1,best_iou)) torch.save(state,save_path) print("checkpoint saved success") writer.close()
def build_loss_and_optimizer(self): # # Classification Loss - SparseSoftmaxCrossEntropyLoss, WeightedSigmoidClassificationLoss # self.cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits( logits=self.logits, labels=self.y) self.cross_entropy_loss = tf.reduce_mean(self.cross_entropy) self.cls_loss = self.cross_entropy_loss self.pred_class = tf.argmax(self.logits, axis=1) self.corrections = tf.equal(self.y, self.pred_class) self.cls_accuracy = tf.reduce_sum(tf.cast( self.corrections, tf.int32)) / tf.shape(self.y)[0] # self.accuracy = tf.metrics.accuracy(labels=self.y, # predictions=tf.argmax(self.logits, axis=1))[1] # # Localisation Loss - WeightedSmoothL1LocalizationLoss # # top-1 loc, pred_class = tf.argmax(self.logits, axis=1) self.cam_t1 = self.get_classmap(pred_class, self.conv) self.pred_bbox, self.debug = utils.get_bbox_from_cam(self.cam_t1, 0.2) self.huber_loss_t1 = tf.losses.huber_loss( self.bbox, self.pred_bbox) # fixme: pred_cls 가 gt_cls 가 아닌 경우 loss 계산 다르게 해야함 self.loc_loss_t1 = tf.reduce_mean(self.huber_loss_t1) self.iou_t1 = utils.calc_iou(self.pred_bbox, self.bbox) t1_iou_over_50 = self.iou_t1 > 0.5 self.correct_and_iou_t1 = tf.logical_and(self.corrections, t1_iou_over_50) self.top1_loc_accuracy = tf.reduce_sum( tf.cast(self.correct_and_iou_t1, tf.float32)) / self.config.bs # gt-known loc, use this! self.cam_gt = self.get_classmap(self.y, self.conv) self.pred_bbox_gt, self.debug_gt = utils.get_bbox_from_cam( self.cam_gt, 0.2) self.huber_loss_gt = tf.losses.huber_loss(self.bbox, self.pred_bbox_gt) self.loc_loss_gt = tf.reduce_mean(self.huber_loss_gt) self.iou_gt = utils.calc_iou(self.pred_bbox_gt, self.bbox) gt_iou_over_50 = self.iou_gt > 0.5 self.correct_and_iou_gt = gt_iou_over_50 self.gt_known_loc_accuracy = tf.reduce_sum( tf.cast(self.correct_and_iou_gt, tf.float32)) / self.config.bs # Total Loss -- loc_loss_gt alpha = self.config.alpha beta = self.config.beta gamma = self.config.gamma # self.tot_loss = alpha * self.cls_loss + beta * self.loc_loss_t1 self.tot_loss = alpha * self.cls_loss + beta * self.loc_loss_gt + gamma * self.loc_loss_t1 # Optimizer if self.config.opt == 'adam': self.optimizer = tf.train.AdamOptimizer(self.learning_rate, beta1=self.config.beta1) elif self.config.opt == 'rmsprop': self.optimizer = tf.train.RMSPropOptimizer(self.learning_rate) # grad-clipping var_list = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, scope=self.model_name) gvs = self.optimizer.compute_gradients(self.tot_loss, var_list=var_list) capped_gvs = [(tf.clip_by_value(grad, -5., 5.), var) for grad, var in gvs] self.train_op = self.optimizer.apply_gradients(capped_gvs) cls_gvs = self.optimizer.compute_gradients(self.cls_loss, var_list=var_list) cls_capped_gvs = [(tf.clip_by_value(grad, -5., 5.), var) for grad, var in cls_gvs] self.cls_train_op = self.optimizer.apply_gradients(cls_capped_gvs)
def train(run, epochs): run_file = open(os.path.join(opt.outputFolder, f"run_{run}.csv"), "w") epoch_loss = [] val_loss = [] acc = [] miou = [] train_time = 0 ce_loss = nn.CrossEntropyLoss() optim = torch.optim.Adam(model.parameters(), lr=opt.lr) start_time = time.time() for i in range(epochs): ep = 0 model.train() for X, y in train_loader: optim.zero_grad() X = X.to(device) y = y.to(device) output = model(X)['out'] loss = ce_loss(output, y) loss.backward() ep += loss.item() optim.step() epoch_loss.append(ep) correct = 0 total = 0 val = 0 iou = [] model.eval() for X, y in eval_loader: X = X.to(device) y = y.to(device) output = model(X)['out'] loss = ce_loss(output, y) val += loss.item() probs = torch.functional.F.softmax(output, 1) y_pred = torch.argmax(probs, dim=1) correct += torch.sum(y_pred == y).item() total += (y.shape[-1] * y.shape[-2]) iou.append(calc_iou(y_pred, y)) val_loss.append(val) miou.append(np.mean(iou)) acc.append(correct / total) print(i, "mIoU: ", round(miou[-1], 2), " - Accuracy: ", round(acc[-1], 2), " - Loss: ", round(val, 1)) run_file.write(f'{i},{round(miou[-1], 2)},{round(val,2)}\n') # early stopping if must_stop(miou): print('***** Early stopping! :)') break train_time += time.time() - start_time print(f"Train_time={train_time/60} (min)") plt.subplots(figsize=(15, 8)) plt.imsave(os.path.join(opt.outputFolder, f"predicition_{run}.png"), np.hstack((y_pred.squeeze().cpu(), y.squeeze().cpu()))) save_file(f'val_loss_{run}.txt', val_loss) save_file(f'epoch_loss_{run}.txt', epoch_loss) save_file(f'acurracy_{run}.txt', acc) save_file(f'miou_{run}.txt', miou) run_file.close() torch.save(model.state_dict(), os.path.join(opt.outputFolder, f'model_{run}.pth')) return val_loss[-1], epoch_loss[-1], acc[-1], miou[-1]
def main(args): vis = args.vis debug = args.debug save = args.save nparticles = args.particles root_path = '/home/ccjiang/Documents/caffe-fast-rcnn/examples/tracker/' dataset_path = args.dataset #"/data/OTB100" dataset100_seq = [ 'Bird2', 'BlurCar1', 'BlurCar3', 'BlurCar4', 'Board', 'Bolt2', 'Boy', 'Car2', 'Car24', 'Coke', 'Coupon', 'Crossing', 'Dancer', 'Dancer2', 'David2', 'David3', 'Dog', 'Dog1', 'Doll', 'FaceOcc1', 'FaceOcc2', 'Fish', 'FleetFace', 'Football1', 'Freeman1', 'Freeman3', 'Girl2', 'Gym', 'Human2', 'Human5', 'Human7', 'Human8', 'Jogging', 'KiteSurf', 'Lemming', 'Man', 'Mhyang', 'MountainBike', 'Rubik', 'Singer1', 'Skater', 'Skater2', 'Subway', 'Suv', 'Tiger1', 'Toy', 'Trans', 'Twinnings', 'Vase' ] dataset50_seq = [ 'Basketball', 'Biker', 'Bird1', 'BlurBody', 'BlurCar2', 'BlurFace', 'BlurOwl', 'Bolt', 'Box', 'Car1', 'Car4', 'CarDark', 'CarScale', 'ClifBar', 'Couple', 'Crowds', 'Deer', 'Diving', 'DragonBaby', 'Dudek', 'Football', 'Freeman4', 'Girl', 'Human3', 'Human4', 'Human6', 'Human9', 'Ironman', 'Jump', 'Jumping', 'Liquor', 'Matrix', 'MotorRolling', 'Panda', 'RedTeam', 'Shaking', 'Singer2', 'Skating1', 'Skating2', 'Skiing', 'Soccer', 'Surfer', 'Sylvester', 'Tiger2', 'Trellis', 'Walking', 'Walking2', 'Woman' ] if "OTB50" in dataset_path: data_seq = dataset50_seq else: data_seq = dataset100_seq log_name = 'log.txt' log_file = open(log_name, 'w') records_success = [] #defaultdict(list) records_precision = [] #defaultdict(list) records_reinit = defaultdict(list) model_def = os.path.join(root_path, args.prototxt) model_weight = os.path.join(root_path, args.caffemodel) vggnet = VGGnet.VGGnet(model_def, model_weight) thre_max_neg = 0.3 # 0.5 test_times = 1 # 0 for t in range(test_times): print 'Test round: %d' % t log_file.write('Test round: %d\n' % t) # sequences = ['Fish'] for sequence in data_seq: for t in os.walk( os.path.join(dataset_path, sequence, sequence, "img")): if t[0] == os.path.join(dataset_path, sequence, sequence, "img"): nFrame = len(t[2]) print 'Processing: %s' % sequence log_file.write('Processing: %s\n' % sequence) print "Total frames are: ", nFrame log_file.write('Total frames are: %d\n' % nFrame) gt_path = os.path.join(dataset_path, sequence, sequence, "groundtruth_rect.txt") gt_boxes = utils.get_boxes_all(gt_path) conf_hist = [] iou_hist = [] area_hist = [] pred_hist = [] # (x1,y1,x2,y2) eig_hist = [] reinit = 0 nFrame = np.minimum(nFrame, gt_boxes.shape[0]) id_shift = 0 init_id = False for id in np.arange(0, nFrame): frame_name = "img/%04d.jpg" % (id + 1) # print "Start processing: %s" % frame_name frame_path = os.path.join(dataset_path, sequence, sequence, frame_name) if os.path.exists(frame_path) == False: id_shift = id_shift + 1 continue id = id - id_shift frame_data = caffe.io.load_image( frame_path) # (432,576,3), in [0,1] gt_box = gt_boxes[id] if init_id == False: h, w, c = frame_data.shape frame_shape = [c, w, h] fail_times = 0 area = (gt_box[2] - gt_box[0]) * (gt_box[3] - gt_box[1]) ratio = (gt_box[2] - gt_box[0]) / (gt_box[3] - gt_box[1] ) # ratio=w/h # set up net.blobs['im_info'] print "Image Size: ", w, h log_file.write('Image Size: %d %d\n' % (w, h)) vggnet.reshape(w=w, h=h, nbox=nparticles) filter = PFfilter.PFfilter( utils.bbox_to_states(gt_box, area, ratio), area, ratio, w, h, nparticles) filter.create_particles() filter.restrict_particles(w, h) area_hist.append(filter.cur_a) pred_hist.append(np.array(gt_box).reshape(1, -1)) # pca # test sample_iou num_true = 500 num_false = 1000 boxes_train = [] # boxes_train_neg=[] iou_train = [] try: # Q=[[1,0],[0,1]] #for pixel wise Q = 0.05 # box_w,box_h sample_box_true, sample_iou_true = filter.sample_iou_pred_box( gt_box, Q, 0.01, 0.01, num_true, 0.8, 1.0) except OverflowError as e: print "too many loops in sample." # print sample_box_true[:10] # print sample_box_true.shape[0] # print sample_iou_true[:10] # print "average iou: ", np.mean(sample_iou_true) boxes_train.append(sample_box_true) iou_train.append(sample_iou_true) try: # Q=[[36,0],[0,36]]#for pixel wise Q = 0.2 # 0.15 sample_box_false, sample_iou_false = filter.sample_iou( gt_box, Q, 0.2, 0.01, num_false / 2, 0, thre_max_neg) except OverflowError as e: print "too many loops in sample." # print sample_box_false[:10] # print sample_box_false.shape[0] # print sample_iou_false[:10] # print "average iou: ", np.mean(sample_iou_false) boxes_train.append(sample_box_false) iou_train.append(sample_iou_false) try: # Q=[[36,0],[0,36]]#for pixel wise Q = 0.2 sample_box_false, sample_iou_false = filter.sample_iou( gt_box, Q, 0.01, 0.2, num_false / 2, 0, thre_max_neg) except OverflowError as e: print "too many loops in sample." # print sample_box_false[:10] # print sample_box_false.shape[0] # print sample_iou_false[:10] # print "average iou: ", np.mean(sample_iou_false) boxes_train.append(sample_box_false) iou_train.append(sample_iou_false) boxes_train = np.vstack(boxes_train) iou_train = np.vstack(iou_train) y_train_true = np.ones((num_true, )) y_train_false = np.zeros((num_false, )) y_train = np.hstack([y_train_true, y_train_false]) # permutation ind_perm = np.random.permutation( range(num_false + num_true)) boxes_train = boxes_train[ind_perm, :] iou_train = iou_train[ind_perm] y_train = y_train[ind_perm] ind_pos = np.where(y_train == 1)[0] ind_neg = np.where(y_train == 0)[0] vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0]) features = vggnet.get_features_first_raw( frame_data, boxes_raw=boxes_train, id=id) for k, v in features.iteritems(): # print k,v.shape if k == 'f3': pca3 = utils.skl_pca(v) v_pca3 = pca3.transform(v) pca3_pos = np.zeros((num_true, pca3.n_components_), dtype=np.float32) pca3_neg = np.zeros( (num_false, pca3.n_components_), dtype=np.float32) pca3_pos[...] = v_pca3[ind_pos, :] pca3_neg[...] = v_pca3[ind_neg, :] # utils.vis_as_image(v_pca3) # plt.imshow(v_pca3) # plt.title("PCA features") # plt.show() # plt.close() # logistic regression y_weight = sklearn.utils.class_weight.compute_class_weight( class_weight='balanced', classes=np.array([0, 1]), y=y_train) # print y_weight class_weight = {0: y_weight[0], 1: y_weight[1]} clf3 = linear_model.LogisticRegression( fit_intercept=True, solver='liblinear') clf3.fit(v_pca3, y_train) vis_feature = False if vis_feature: utils.vis_features(features, id) start_time = time.time() else: if fail_times >= 5: # reinitialize reinit += 1 area = (gt_box[2] - gt_box[0]) * (gt_box[3] - gt_box[1]) ratio = (gt_box[2] - gt_box[0]) / (gt_box[3] - gt_box[1]) filter = PFfilter.PFfilter( utils.bbox_to_states(gt_box, area, ratio), area, ratio, w, h, nparticles) # filter.reset(utils.bbox_to_states(gt_box, area, ratio), area, ratio) filter.create_particles() filter.restrict_particles(w, h) area_hist.append(filter.cur_a) pred_box = gt_box pred_hist.append(np.array(pred_box).reshape(1, -1)) conf_hist.append(-0.1) boxes_train = [] # boxes_train_neg=[] iou_train = [] try: # Q=[[1,0],[0,1]] #for pixel wise Q = 0.05 # box_w,box_h sample_box_true, sample_iou_true = filter.sample_iou_pred_box( gt_box, Q, 0.01, 0.01, num_true, 0.8, 1.0) except OverflowError as e: print "too many loops in sample." boxes_train.append(sample_box_true) iou_train.append(sample_iou_true) try: # Q=[[36,0],[0,36]]#for pixel wise Q = 0.2 # 0.15 sample_box_false, sample_iou_false = filter.sample_iou( gt_box, Q, 0.2, 0.01, num_false / 2, 0, thre_max_neg) except OverflowError as e: print "too many loops in sample." boxes_train.append(sample_box_false) iou_train.append(sample_iou_false) try: # Q=[[36,0],[0,36]]#for pixel wise Q = 0.2 sample_box_false, sample_iou_false = filter.sample_iou( gt_box, Q, 0.01, 0.2, num_false / 2, 0, thre_max_neg) except OverflowError as e: print "too many loops in sample." # print sample_box_false[:10] # print sample_box_false.shape[0] # print sample_iou_false[:10] # print "average iou: ", np.mean(sample_iou_false) boxes_train.append(sample_box_false) iou_train.append(sample_iou_false) boxes_train = np.vstack(boxes_train) iou_train = np.vstack(iou_train) y_train_true = np.ones((num_true, )) y_train_false = np.zeros((num_false, )) y_train = np.hstack([y_train_true, y_train_false]) # permutation ind_perm = np.random.permutation( range(num_false + num_true)) boxes_train = boxes_train[ind_perm, :] iou_train = iou_train[ind_perm] y_train = y_train[ind_perm] ind_pos = np.where(y_train == 1)[0] ind_neg = np.where(y_train == 0)[0] vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0]) features = vggnet.get_features_first_raw( frame_data, boxes_raw=boxes_train, id=id) for k, v in features.iteritems(): # print k, v.shape if k == 'f3': v_pca3 = pca3.transform(v) pca3_pos[...] = v_pca3[ind_pos, :] pca3_neg[...] = v_pca3[ind_neg, :] clf3.fit(v_pca3, y_train) score3 = clf3.score(v_pca3, y_train) # print 'score3: ', score3 # prob=clf3.predict_proba(v_pca3) # print clf3.classes_ fail_times = 0 continue filter.predict_particles(Q=0.2, cr=0.005, ca=0.001) filter.restrict_particles(w, h) area_hist.append(filter.cur_a) # compute conf conf = np.zeros(filter.weights.shape) # np.save('particles.npy',filter.particles) pred_boxes = utils.state_to_bbox(filter.particles, area, ratio) vggnet.reshape(w, h, filter.num_particles) features = vggnet.get_features_first_raw( frame_data, boxes_raw=pred_boxes, id=id) for k, v in features.iteritems(): # print k,v.shape if k == 'f3': v_pca3 = pca3.transform(v) # utils.vis_as_image(v_pca3) # plt.imshow(v_pca3) # plt.title("PCA features") # plt.show() # plt.close() # logistic regression conf = clf3.predict_proba(v_pca3)[:, 1] conf_max = np.max(conf) conf_min = np.min(conf) filter.update_particles(conf) filter.resample() # always resample pred_state, s_particles, r_particles = filter.estimate( k=10) cov_particles = np.dot( filter.particles[:, :2].T, filter.particles[:, :2]) / filter.particles.shape[0] eigval, eigvec = np.linalg.eig(cov_particles) max_val = eigval[0] eig_hist.append(max_val) print 'Max eigvalue: %f' % max_val # print 'conf is: ',conf if conf_max > 0.8: fail_times = 0 else: fail_times += 1 print "conf_max too low, not update particles " pred_box = utils.state_to_bbox(pred_state.reshape((-1, 6)), area, ratio) show_sr = False if show_sr: count, xedge, yedge, tmp_im = plt.hist2d( s_particles, r_particles, bins=10, weights=filter.weights.squeeze(), cmap=plt.cm.gray) top3 = np.argsort(-count, axis=None)[:3] row_ind = top3[:] / count.shape[1] col_ind = top3[:] % count.shape[0] ''' plt.scatter(s_particles,r_particles,c='r',marker='.',linewidths=1) plt.xlabel('Area') plt.ylabel('Aspect ratio') plt.title('Area and Ratio of particles') plt.axis('equal') ''' plt.show() iou = utils.calc_iou(gt_box, pred_box) # print 'iou is: ', iou pred_hist.append(pred_box) conf_hist.append(conf_max) iou_hist.append(iou) if conf_max >= 0.9: # 0.8 # update pca3_pos and pca3_neg new_true = 100 # 50 new_false = 200 # 100 boxes_train = [] iou_train = [] Q = 0.02 try: sample_box_true, sample_iou_true = filter.sample_iou_pred_box( pred_box, Q, 0.01, 0.01, new_true, 0.85, 1.0) except OverflowError as e: print "too many loops in sample." # print sample_box_true[:10] # print sample_box_true.shape[0] # print sample_iou_true[:10] # print "average iou: ", np.mean(sample_iou_true) boxes_train.append(sample_box_true) iou_train.append(sample_iou_true) try: Q = 0.2 sample_box_false, sample_iou_false = filter.sample_iou( pred_box, Q, 0.2, 0.01, new_false / 2, 0, thre_max_neg) except OverflowError as e: print "too many loops in sample." # print sample_box_false[:10] # print sample_box_false.shape[0] # print sample_iou_false[:10] # print "average iou: ", np.mean(sample_iou_false) boxes_train.append(sample_box_false) iou_train.append(sample_iou_false) try: Q = 0.2 sample_box_false, sample_iou_false = filter.sample_iou( pred_box, Q, 0.01, 0.2, new_false / 2, 0, thre_max_neg) except OverflowError as e: print "too many loops in sample." boxes_train.append(sample_box_false) iou_train.append(sample_iou_false) boxes_train = np.vstack(boxes_train) iou_train = np.vstack(iou_train) y_train_true = np.ones((new_true, )) y_train_false = np.zeros((new_false, )) y_train = np.hstack([y_train_true, y_train_false]) # permutation ind_perm = np.random.permutation( range(new_false + new_true)) boxes_train = boxes_train[ind_perm, :] y_train = y_train[ind_perm] new_y = np.zeros(y_train.shape) new_y[...] = y_train ind_pos = np.where(y_train == 1)[0] ind_neg = np.where(y_train == 0)[0] vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0]) features = vggnet.get_features_second_raw( boxes_raw=boxes_train, id=id) for k, v in features.iteritems(): # print k, v.shape if k == 'f3': v_pca3 = pca3.transform(v) # random substitude pca3_cur_pos = v_pca3[ind_pos, :] pca3_cur_neg = v_pca3[ind_neg, :] to_subst = random.sample( range(num_true), new_true) pca3_pos[to_subst, :] = pca3_cur_pos to_subst = random.sample( range(num_false), new_false) pca3_neg[to_subst, :] = pca3_cur_neg if conf_max < 1 and fail_times >= 2: pca3_train = np.vstack([pca3_pos, pca3_neg]) y_train_true = np.ones((num_true, )) y_train_false = np.zeros((num_false, )) y_train = np.hstack([y_train_true, y_train_false]) # permutation ind_perm = np.random.permutation( range(num_false + num_true)) pca3_train = pca3_train[ind_perm, :] y_train = y_train[ind_perm] # logistic regression clf3.fit(pca3_train, y_train) # print 'score is: ',clf3.score(pca3_train,y_train) # (B,G,R) frame_data_cv = frame_data * 255 # [0,1]-->[0,255] frame_data_cv = frame_data_cv[:, :, ::-1] # RGB->BGR frame_data_cv = frame_data_cv.astype('uint8') cv2.rectangle(frame_data_cv, (int(gt_box[0]), int(gt_box[1])), (int(gt_box[2]), int(gt_box[3])), (255, 0, 0), 2, 1) if id > 0 and init_id == True: cv2.rectangle(frame_data_cv, (int(pred_box[0, 0]), int(pred_box[0, 1])), (int(pred_box[0, 2]), int(pred_box[0, 3])), (0, 255, 0), 2, 1) if init_id == False: init_id = True show_particles = False if show_particles: for i in range(filter.num_particles): cx = filter.particles[i, 0] cy = filter.particles[i, 1] cv2.circle(frame_data_cv, (int(cx), int(cy)), 1, (0, 0, 255), thickness=1) show_box = False if show_box: n = 0 for i in ind_pos: if n % 5 == 0: cv2.rectangle(frame_data_cv, (int( boxes_train[i, 0]), int(boxes_train[i, 1])), (int(boxes_train[i, 2]), int(boxes_train[i, 3])), (0, 0, 255), 2, 1) n += 1 n = 0 ''' for i in ind_neg: if n%15==0: cv2.rectangle(frame_data_cv, (int(boxes_train[i, 0]), int(boxes_train[i, 1])), (int(boxes_train[i, 2]), int(boxes_train[i, 3])), (0, 255,255), 2, 1) n+=1 ''' show_particles_init = False if show_particles_init: for i in range(filter.num_particles): cx = filter.particles[i, 0] cy = filter.particles[i, 1] cv2.circle(frame_data_cv, (int(cx), int(cy)), 1, (0, 255, 0), thickness=1) show_frame = False cv2.circle(frame_data_cv, (int(filter.cur_c[0]), int(filter.cur_c[1])), 2, (0, 0, 255), thickness=1) if show_frame: cv2.imshow(sequence, frame_data_cv) c = cv2.waitKey(1) # print 'You press: ',chr(c) # if chr(c)=='c': if c != -1: cv2.destroyWindow(sequence) # conf_hist=np.array(conf_hist) # iou_hist=np.array(iou_hist) # np.save('conf_hist.npy',conf_hist) # np.save('iou_hist.npy',iou_hist) break end_time = time.time() print "Average FPS: %f" % (nFrame / (end_time - start_time)) log_file.write("Average FPS: %f\n" % (nFrame / (end_time - start_time))) conf_hist = np.array(conf_hist) iou_hist = np.array(iou_hist) area_hist = np.array(area_hist) pred_hist = np.vstack(pred_hist) precisions, auc_pre = utils.calc_prec(gt_boxes, pred_hist) # plt.figure() # plt.subplot(221) # plt.plot(precisions) # plt.gca().invert_xaxis() # plt.title("Precision plot") # plt.xlabel('Location error threshold') # plt.ylabel('Precision') # plt.yticks(np.linspace(0,1,11)) # plt.subplot(222) # plt.show() suc, auc_iou = utils.calc_success(iou_hist) records_precision.append(precisions * nFrame) records_success.append(suc * nFrame) # plt.plot(suc) # plt.gca().invert_xaxis() # plt.title('Success plot') # plt.xlabel('Overlap threshold') # plt.ylabel('Success Rate') # plt.yticks(np.linspace(0,1,11)) # plt.show() # np.save('conf_hist.npy', conf_hist) # np.save('iou_hist.npy', iou_hist) # np.save('area_hist.npy',area_hist) # print 'Average iou is: %f'%(np.mean(iou_hist)) print 'Precision @20 is: %f' % precisions[19] print 'Auc of Precision is: %f' % auc_pre print 'Auc of Success is: %f' % auc_iou print 'Reinit times: %d' % reinit log_file.write("Precision @20 is: %f\n" % precisions[19]) log_file.write('Auc of Precision is: %f\n' % auc_pre) log_file.write('Auc of Success is: %f\n' % auc_iou) log_file.write('Reinit times: %d\n' % reinit) log_file.close() pkl = open( '/home/ccjiang/Documents/caffe-fast-rcnn/examples/tracker/results_100.pkl', 'w') pickle.dump([records_precision, records_success], pkl) pkl.close()