Example #1
0
device = 'cuda' if torch.cuda.is_available() else 'cpu'

if __name__ == "__main__":
    os.environ['CUDA_VISIBLE_DEVICES'] = '1'
    torch.manual_seed(777)
    if device == 'cuda':
        torch.cuda.manual_seed_all(777)

    path = './Magic_card/origin_data'
    # load data
    train_data = D_set.load_data(path)
    data_loader = D_set.custom_loader(train_data, 8)
    #data_loader = torch.utils.data.DataLoader(dataset = train_data,batch_size = 8, shuffle =True, num_workers =2)
    #setting loss_func, net, optim
    conv = VGG_n.make_layers(VGG_n.cfg['E'])
    net = VGG_n.VGG(conv, num_classes=2, init_weights=True)
    model = net.to(device)

    optimizer = torch.optim.Adam(model.parameters(), lr=0.00005)
    loss_func = nn.CrossEntropyLoss().to(device)
    total_batch = len(data_loader)

    epochs = 10
    model.train
    for epoch in range(epochs):
        avg_cost = 0
        for X, Y in data_loader:
            X = X.to(device)
            Y = Y.to(device)
def main(args):
    vis = args.vis
    debug = args.debug
    save = args.save
    nparticles = args.particles
    root_path = '/home/ccjiang/Documents/py-faster-rcnn/caffe-fast-rcnn/examples/tracker/'
    dataset_path = args.dataset  # "/data/OTB100"
    dataset100_seq = ['Bird2', 'BlurCar1', 'BlurCar3', 'BlurCar4', 'Board', 'Bolt2', 'Boy', 'Car2',
                      'Car24', 'Coke', 'Coupon', 'Crossing', 'Dancer', 'Dancer2', 'David2', 'David3',
                      'Dog', 'Dog1', 'Doll', 'FaceOcc1', 'FaceOcc2', 'Fish', 'FleetFace', 'Football1',
                      'Freeman1', 'Freeman3', 'Girl2', 'Gym', 'Human2', 'Human5', 'Human7', 'Human8',
                      'Jogging', 'KiteSurf', 'Lemming', 'Man', 'Mhyang', 'MountainBike', 'Rubik',
                      'Singer1', 'Skater', 'Skater2', 'Subway', 'Suv', 'Tiger1', 'Toy', 'Trans',
                      'Twinnings', 'Vase']
    dataset50_seq = ['Basketball',  'Bird1', 'BlurBody', 'BlurCar2', 'BlurFace', 'BlurOwl',
                     'Bolt', 'Box', 'Car1', 'Car4', 'CarDark', 'CarScale', 'ClifBar', 'Couple', 'Crowds','David',
                     'Deer', 'Diving', 'DragonBaby', 'Dudek', 'Football', 'Freeman4', 'Girl',
                     'Human3', 'Human4', 'Human6', 'Human9', 'Ironman', 'Jump', 'Jumping', 'Liquor',
                     'Matrix', 'MotorRolling', 'Panda', 'RedTeam', 'Shaking', 'Singer2', 'Skating1',
                     'Skating2', 'Skiing', 'Soccer', 'Surfer', 'Sylvester', 'Tiger2', 'Trellis',
                     'Walking', 'Walking2', 'Woman']
    datafull_seq = dataset100_seq + dataset50_seq
    if "OTB50" in dataset_path:
        data_seq = dataset50_seq
    else:
        data_seq = dataset100_seq

    log_name = 'log_1119.txt'
    log_file = open(log_name, 'w')
    records_success = []  # defaultdict(list)
    records_precision = []  # defaultdict(list)
    records_reinit = defaultdict(list)
    model_def = os.path.join(root_path, args.prototxt)
    model_weight = os.path.join(root_path, args.caffemodel)
    vggnet = VGGnet.VGGnet(model_def, model_weight)

    thre_max_neg = 0.3  # 0.5
    test_times = 1  # 0
    for t in range(test_times):
        print 'Test round: %d' % t
        log_file.write('Test round: %d\n' % t)
        # sequences = ['Fish']
        for sequence in datafull_seq:  # datafull_seq
            if sequence in dataset50_seq:
                dataset_path = "/data/OTB50"
            else:
                dataset_path = "/data/OTB100"
            for t in os.walk(os.path.join(dataset_path, sequence, sequence, "img")):
                if t[0] == os.path.join(dataset_path, sequence, sequence, "img"):
                    nFrame = len(t[2])
                    print 'Processing: %s' % sequence
                    log_file.write('Processing: %s\n' % sequence)
                    print "Total frames are: ", nFrame
                    log_file.write('Total frames are: %d\n' % nFrame)
            gt_path = os.path.join(dataset_path, sequence, sequence, "groundtruth_rect.txt")

            gt_boxes = utils.get_boxes_all(gt_path)

            conf_hist = []
            iou_hist = []
            area_hist = []
            pred_hist = []  # (x1,y1,x2,y2)
            eig_hist = []
            reinit = 0
            nFrame = np.minimum(nFrame, gt_boxes.shape[0])

            id_shift = 0
            init_id = False
            update_recent = False
            for id in np.arange(0, nFrame):
                frame_name = "img/%04d.jpg" % (id + 1)
                # print "Start processing: %s" % frame_name
                frame_path = os.path.join(dataset_path, sequence, sequence, frame_name)
                if os.path.exists(frame_path) == False:
                    id_shift = id_shift + 1
                    continue
                id = id - id_shift

                frame_data = caffe.io.load_image(frame_path)  # (432,576,3), in [0,1]
                gt_box = gt_boxes[id]

                if init_id == False:
                    h, w, c = frame_data.shape
                    frame_shape = [c, w, h]
                    fps = 20
                    fourcc = cv2.VideoWriter_fourcc(*'MJPG')
                    video_writer = cv2.VideoWriter("res_%s.avi"%sequence,fourcc,fps,(w,h))
                    fail_times = 0
                    box_w = gt_box[2] - gt_box[0]
                    box_h = gt_box[3] - gt_box[1]
                    area = (gt_box[2] - gt_box[0]) * (gt_box[3] - gt_box[1])
                    ratio = (gt_box[2] - gt_box[0]) / (gt_box[3] - gt_box[1])  # ratio=w/h
                    # set up net.blobs['im_info']
                    print "Image Size: ", w, h
                    log_file.write('Image Size: %d %d\n' % (w, h))
                    b = gt_box[np.newaxis, :]
                    vggnet.reshape(w=w, h=h, nbox=b.shape[0])
                    features0 = vggnet.get_features("conv3_3", frame_data,
                                                    boxes_raw=b)  # shape:(256,hs,ws),conv3_3,res3b3
                    features0 = np.squeeze(features0)
                    pca_f, scaler_f = featmap_pca2(features0,ncompnents=128)#128

                    box_w = gt_box[2] - gt_box[0]
                    box_h = gt_box[3] - gt_box[1]

                    vggnet.reshape(w=w, h=h, nbox=nparticles)
                    pfilter = PFfilter.PFfilter(utils.bbox_to_states(gt_box, area, ratio), area, ratio, w, h,
                                                nparticles)
                    pfilter.create_particles()
                    pfilter.restrict_particles(w, h)
                    area_hist.append(pfilter.cur_a)
                    pred_hist.append(np.array(gt_box).reshape(1, -1))
                    # pca
                    # test sample_iou
                    num_true = 500
                    num_false = 1000  # 1000
                    #data augument
                    gt_box_otb = gt_box.copy()
                    gt_box_otb[2] -= gt_box_otb[0]
                    gt_box_otb[3] -= gt_box_otb[1]
                    boxes_train = []
                    ids = np.zeros(num_false + num_true)


                    imgs = []
                    for i in np.arange(4):
                        if i == 0:
                            img1,gt1,img2,gt2,img3,gt3 = DataAugment(frame_data,gt_box_otb,True)
                            gt1[2] += gt1[0]
                            gt1[3] += gt1[1]
                            gt2[2] += gt2[0]
                            gt2[3] += gt2[1]
                            gt3[2] += gt3[0]
                            gt3[3] += gt3[1]
                            box_true1, iou_true = pfilter.sample_iou_pred_box(gt1, 0.05, 0.01, 0.01, 20, 0.8, 1.0)
                            box_true2, iou_true = pfilter.sample_iou_pred_box(gt2, 0.05, 0.01, 0.01, 40, 0.8, 1.0)
                            box_true3, iou_true = pfilter.sample_iou_pred_box(gt3, 0.05, 0.01, 0.01, 20, 0.8, 1.0)
                            box_true1[0, ...] = gt1
                            box_true2[0, ...] = gt2
                            box_true3[0, ...] = gt3

                            boxes_train.append(box_true1)
                            boxes_train.append(box_true2)
                            boxes_train.append(box_true3)
                            imgs.append(img1)
                            imgs.append(img2)
                            imgs.append(img3)
                            ids[20:60] = 1
                            ids[60:80] = 2

                        else:
                            img1, gt1, img2, gt2 = DataAugment(frame_data, gt_box_otb,False)
                            gt1[2] += gt1[0]
                            gt1[3] += gt1[1]
                            gt2[2] += gt2[0]
                            gt2[3] += gt2[1]

                            box_true1, iou_true = pfilter.sample_iou_pred_box(gt1, 0.05, 0.01, 0.01, 20, 0.8, 1.0)
                            box_true2, iou_true = pfilter.sample_iou_pred_box(gt2, 0.05, 0.01, 0.01, 20, 0.8, 1.0)

                            box_true1[0, ...] = gt1
                            box_true2[0, ...] = gt2

                            boxes_train.append(box_true1)
                            boxes_train.append(box_true2)
                            imgs.append(img1)
                            imgs.append(img2)
                            cur_i = 80+(i-1)*40
                            ids[cur_i:(cur_i+20)] = 3+(i-1)*2
                            ids[(cur_i+20):(cur_i+40)] = 3+(i-1)*2+1
                    # boxes_train_neg=[]

                    try:
                        # Q=[[1,0],[0,1]] #for pixel wise
                        Q = 0.05  # box_w,box_h,0.05
                        sample_box_true, sample_iou_true = pfilter.sample_iou_pred_box(gt_box, Q, 0.01, 0.01,
                                                                                       num_true-200, 0.8,
                                                                                       1.0)#0.8
                    except OverflowError as e:
                        print "too many loops in sample in Initialize--TRUE."

                    boxes_train.append(sample_box_true)

                    try:
                        # Q=[[36,0],[0,36]]#for pixel wise
                        Q = 0.2  # 0.2
                        sample_box_false, sample_iou_false = pfilter.sample_iou(gt_box, Q, 0.2, 0.01, num_false / 2, 0,
                                                                                thre_max_neg)  # 0.2,0.01
                    except OverflowError as e:
                        print "too many loops in sample in Initialize--FALSE."
                    # print sample_box_false[:10]
                    # print sample_box_false.shape[0]
                    # print sample_iou_false[:10]
                    # print "average iou: ", np.mean(sample_iou_false)
                    boxes_train.append(sample_box_false)

                    try:
                        # Q=[[36,0],[0,36]]#for pixel wise
                        Q = 0.2  # 0.2
                        sample_box_false, sample_iou_false = pfilter.sample_iou(gt_box, Q, 0.01, 0.2, num_false / 2, 0,
                                                                                thre_max_neg)  # 0.01,0.2
                    except OverflowError as e:
                        print "too many loops in sample in Initialize--FALSE."

                    boxes_train.append(sample_box_false)

                    boxes_train = np.vstack(boxes_train)

                    imgs.append(frame_data)
                    imgs = np.stack(imgs,axis=0)#(10,h,w,c)
                    ids[200:] = 9
                    y_train_true = np.ones((num_true,))
                    y_train_false = np.zeros((num_false,))
                    ids_save = np.ones((num_true+num_false))
                    ids_save[num_true:] = 0
                    ids_save[20:60] = 2
                    y_train = np.hstack([y_train_true, y_train_false])

                    # permutation
                    ind_perm = np.random.permutation(range(num_false + num_true))
                    boxes_train = boxes_train[ind_perm, :]
                    ids_save = ids_save[ind_perm]
                    y_train = y_train[ind_perm]
                    ids = ids[ind_perm]
                    ind_pos = np.where(y_train == 1)[0]
                    ind_neg = np.where(y_train == 0)[0]

                    vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0],batch_size=10)
                    #features = vggnet.get_features_first_raw(frame_data, boxes_raw=boxes_train, id=id)
                    features = vggnet.get_features_first_id(imgs,boxes_raw=boxes_train,id=ids)
                    #features = vggnet.get_features_first_sel(frame_data, boxes_raw=boxes_train, id=id, sel=f_inds)
                    for k, v in features.iteritems():
                        # print k,v.shape
                        if k == 'f3':
                            #pca3, scaler1, nPCA = utils.skl_pca2(v)
                            v = feat_transformpca(pca_f, scaler_f, v)  # (N,128,7,7)
                            #pca3,scaler,nPCA=utils.skl_pca2(v)
                            #v_pca3 = pca3.transform(scaler.transform(v))
                            #np.save("pca_results/testpca_%s.npy"%sequence,v_pca3)
                            #np.save('labelpca.npy',y_train)
                            #np.save("pca_results/label_pca_%s"%sequence,ids_save)
                            pca3_pos = np.zeros((num_true, pca_f.n_components_*49), dtype=np.float32)
                            pca3_neg = np.zeros((num_false, pca_f.n_components_*49), dtype=np.float32)
                            pca3_pos[...] = v[ind_pos, :]
                            pca3_neg[...] = v[ind_neg, :]
                            # utils.vis_as_image(v_pca3)
                            # plt.imshow(v_pca3)
                            # plt.title("PCA features")
                            # plt.show()
                            # plt.close()
                            # logistic regression
                            y_weight = sklearn.utils.class_weight.compute_class_weight(class_weight='balanced',
                                                                                       classes=np.array([0, 1]),
                                                                                       y=y_train)
                            # print y_weight
                            class_weight = {0: y_weight[0], 1: y_weight[1]}
                            clf3 = linear_model.LogisticRegression(fit_intercept=True, solver='liblinear')
                            clf3.fit(v, y_train)

                    vis_feature = False
                    if vis_feature:
                        utils.vis_features(features, id)

                    start_time = time.time()
                else:
                    if fail_times >= 5:
                        # reinitialize
                        update_recent = False
                        reinit += 1
                        area = (gt_box[2] - gt_box[0]) * (gt_box[3] - gt_box[1])
                        ratio = (gt_box[2] - gt_box[0]) / (gt_box[3] - gt_box[1])
                        pfilter = PFfilter.PFfilter(utils.bbox_to_states(gt_box, area, ratio), area, ratio, w, h,
                                                    nparticles)
                        # filter.reset(utils.bbox_to_states(gt_box, area, ratio), area, ratio)
                        pfilter.create_particles()
                        pfilter.restrict_particles(w, h)
                        area_hist.append(pfilter.cur_a)
                        pred_box = gt_box
                        pred_hist.append(np.array(pred_box).reshape(1, -1))
                        conf_hist.append(-0.1)
                        boxes_train = []
                        # boxes_train_neg=[]
                        iou_train = []
                        try:
                            # Q=[[1,0],[0,1]] #for pixel wise
                            Q = 0.05  # box_w,box_h,0.05
                            sample_box_true, sample_iou_true = pfilter.sample_iou_pred_box(gt_box, Q, 0.01, 0.01,
                                                                                           num_true, 0.8,
                                                                                           1.0)
                        except OverflowError as e:
                            print "too many loops in sample in Reinitialize--TRUE."

                        boxes_train.append(sample_box_true)
                        iou_train.append(sample_iou_true)
                        try:
                            # Q=[[36,0],[0,36]]#for pixel wise
                            Q = 0.2  # 0.2
                            sample_box_false, sample_iou_false = pfilter.sample_iou(gt_box, Q, 0.01, 0.2, num_false / 2,
                                                                                    0,
                                                                                    thre_max_neg)
                        except OverflowError as e:
                            print "too many loops in sample in Reinitialize--FALSE."

                        boxes_train.append(sample_box_false)
                        iou_train.append(sample_iou_false)
                        try:
                            # Q=[[36,0],[0,36]]#for pixel wise
                            Q = 0.2  # 0.2
                            sample_box_false, sample_iou_false = pfilter.sample_iou(gt_box, Q, 0.2, 0.01, num_false / 2,
                                                                                    0,
                                                                                    thre_max_neg)
                        except OverflowError as e:
                            print "too many loops in sample in Reinitialize--FALSE."
                        # print sample_box_false[:10]
                        # print sample_box_false.shape[0]
                        # print sample_iou_false[:10]
                        # print "average iou: ", np.mean(sample_iou_false)
                        boxes_train.append(sample_box_false)
                        iou_train.append(sample_iou_false)

                        boxes_train = np.vstack(boxes_train)
                        iou_train = np.vstack(iou_train)
                        y_train_true = np.ones((num_true,))
                        y_train_false = np.zeros((num_false,))

                        y_train = np.hstack([y_train_true, y_train_false])
                        # permutation
                        ind_perm = np.random.permutation(range(num_false + num_true))
                        boxes_train = boxes_train[ind_perm, :]
                        iou_train = iou_train[ind_perm]
                        y_train = y_train[ind_perm]
                        ind_pos = np.where(y_train == 1)[0]
                        ind_neg = np.where(y_train == 0)[0]

                        vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0])
                        features = vggnet.get_features_first_raw(frame_data, boxes_raw=boxes_train, id=id)
                        #features=feat_transformpca(pca_f,scaler_f,features)
                        #features = vggnet.get_features_first_sel(frame_data, boxes_raw=boxes_train, id=id, sel=f_inds)
                        for k, v in features.iteritems():
                            # print k, v.shape
                            if k == 'f3':
                                v = feat_transformpca(pca_f, scaler_f, v)  # (N,128,7,7)
                                #v_pca3 = pca3.transform(scaler.transform(v))

                                pca3_pos[...] = v[ind_pos, :]
                                pca3_neg[...] = v[ind_neg, :]
                                clf3 = linear_model.LogisticRegression(fit_intercept=True, solver='liblinear')
                                clf3.fit(v, y_train)
                                # score3 = clf3.score(v_pca3, y_train)

                                # print 'score3: ', score3
                                # prob=clf3.predict_proba(v_pca3)
                                # print clf3.classes_
                        fail_times = 0
                        continue

                    pfilter.predict_particles(Q=0.2, cr=0.01, ca=0.01)  # 0.2,0.01
                    pfilter.restrict_particles(w, h)
                    area_hist.append(pfilter.cur_a)
                    # compute conf
                    # conf = np.zeros(pfilter.weights.shape)
                    # np.save('particles.npy',filter.particles)
                    pred_boxes = utils.state_to_bbox(pfilter.particles, area, ratio)
                    #add Gaussian regularization
                    if id>1:
                        gauss_sig = 0.5
                        gauss_w = np.exp(-np.square((pfilter.particles[:,0]-pred_state[0])/(gauss_sig*box_w)/2.0)-np.square((pfilter.particles[:,1]-pred_state[1])/(gauss_sig*box_h)/2.0))
                        pfilter.update_particles(gauss_w)
                        print gauss_w
                    vggnet.reshape(w, h, pfilter.num_particles)
                    features = vggnet.get_features_first_raw(frame_data, boxes_raw=pred_boxes, id=id)
                    #features=feat_transformpca(pca_f,scaler_f,features)
                    #features = vggnet.get_features_first_sel(frame_data, boxes_raw=pred_boxes, id=id, sel=f_inds)
                    for k, v in features.iteritems():
                        # print k,v.shape
                        if k == 'f3':
                            v = feat_transformpca(pca_f, scaler_f, v)  # (N,128,7,7)
                            vf = v
                            #v_pca3 = pca3.transform(scaler.transform(v))
                            conf = clf3.predict_proba(v)[:, 1]

                    # process preds to find out pred_box in terms of confm
                    conf = np.array(conf)

                    conf_max = np.max(conf)
                    conf_min = np.min(conf)
                    pfilter.update_particles(conf)
                    # do resample first or estimate first?
                    # filter.resample()  # always resample
                    pred_state, s_particles, r_particles = pfilter.estimate(k=10)
                    pfilter.resample()
                    pred_box = utils.state_to_bbox(pred_state.reshape((-1, 6)), area, ratio)

                    hard,hard_negv = nms_pred(pred_box,pred_boxes,vf,conf)
                    if hard:
                        hard_negvN = hard_negv.shape[0]
                        #print hard_negv.shape
                    else:
                        hard_negvN = 0

                    avg_pos = np.mean(pfilter.particles[:, :2], axis=0, keepdims=True)
                    # avg_pos[:,0]/=w
                    # avg_pos[:,1]/=h
                    ptls_avg = (pfilter.particles[:, :2] - avg_pos) / np.array([[box_w, box_h]])
                    cov_particles = np.dot(ptls_avg.T, ptls_avg) / pfilter.particles.shape[
                        0]

                    eigval, eigvec = np.linalg.eig(cov_particles)
                    max_val = eigval[0]
                    eig_hist.append(max_val)
                    print 'Max eigvalue: %f' % max_val

                    # print 'conf is: ',conf
                    if conf_max > 0.5:  # 0.8
                        fail_times = 0
                        update_recent = False
                    else:
                        fail_times += 1

                    show_sr = False
                    if show_sr:
                        count, xedge, yedge, tmp_im = plt.hist2d(s_particles, r_particles, bins=10,
                                                                 weights=pfilter.weights.squeeze(), cmap=plt.cm.gray)
                        top3 = np.argsort(-count, axis=None)[:3]
                        row_ind = top3[:] / count.shape[1]
                        col_ind = top3[:] % count.shape[0]

                        plt.show()
                    print pred_box
                    iou = utils.calc_iou(gt_box, pred_box)
                    # print 'iou is: ', iou
                    pred_hist.append(pred_box)
                    conf_hist.append(conf_max)
                    iou_hist.append(iou)

                    if conf_max >= 0.7:  # 0.5
                        # update pca3_pos and pca3_neg
                        new_true = 100  # 100
                        new_false = 400  # 200
                        boxes_train = []

                        iou_train = []
                        Q = 0.05  # 0.02
                        try:
                            sample_box_true, sample_iou_true = pfilter.sample_iou_pred_box(pred_box, Q, 0.01, 0.01,
                                                                                           new_true,
                                                                                           0.85,
                                                                                           1.0)
                        except OverflowError as e:
                            print "too many loops in sample in Update--TRUE."
                        # print sample_box_true[:10]
                        # print sample_box_true.shape[0]
                        # print sample_iou_true[:10]
                        # print "average iou: ", np.mean(sample_iou_true)
                        boxes_train.append(sample_box_true)

                        iou_train.append(sample_iou_true)
                        # part_iou=utils.calc_iou(pred_box,pred_boxes)

                        # ind_iou=np.where(part_iou<0.3)[0]

                        # ind_n=np.minimum(new_false/2,ind_iou.shape[0])
                        # boxes_train.append(pred_boxes[ind_iou[:ind_n],:])
                        # iou_train.append(part_iou[ind_iou])
                        new_false_left = new_false - hard_negvN  # -ind_n
                        try:
                            Q = 0.2  # 0.2
                            sample_box_false, sample_iou_false = pfilter.sample_iou_pred_box(pred_box, Q, 0.2, 0.01,
                                                                                             (new_false_left + 1) / 2,
                                                                                             0, thre_max_neg)
                        except OverflowError as e:
                            print "too many loops in sample in Update--FALSE."
                        # print sample_box_false[:10]
                        # print sample_box_false.shape[0]
                        # print sample_iou_false[:10]
                        # print "average iou: ", np.mean(sample_iou_false)
                        boxes_train.append(sample_box_false)
                        iou_train.append(sample_iou_false)
                        try:
                            Q = 0.2  # 0.2
                            sample_box_false, sample_iou_false = pfilter.sample_iou_pred_box(pred_box, Q, 0.01, 0.2,
                                                                                             new_false_left / 2, 0,
                                                                                             thre_max_neg)
                        except OverflowError as e:
                            print "too many loops in sample in Update--FALSE."
                        boxes_train.append(sample_box_false)
                        iou_train.append(sample_iou_false)

                        boxes_train = np.vstack(boxes_train)

                        # iou_train = np.vstack(iou_train)




                        vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0])
                        features = vggnet.get_features_second_raw(boxes_raw=boxes_train, id=id)
                        #features = feat_transformpca(pca_f,scaler_f,features)
                        #features = vggnet.get_features_second_sel(boxes_raw=boxes_train, id=id, sel=f_inds)
                        for k, v in features.iteritems():
                            # print k, v.shape
                            if k == 'f3':
                                v = feat_transformpca(pca_f, scaler_f, v)  # (N,128,7,7)
                                #v_pca3 = pca3.transform(scaler.transform(v))
                                if hard:
                                    print v.shape
                                    print hard_negv.shape
                                    v = np.vstack([v,hard_negv])
                                y_train_true = np.ones((new_true,))
                                y_train_false = np.zeros((new_false,))
                                y_train = np.hstack([y_train_true, y_train_false])
                                # permutation
                                ind_perm = np.random.permutation(range(new_false + new_true))
                                #boxes_train = boxes_train[ind_perm, :]
                                v = v[ind_perm,:]
                                y_train = y_train[ind_perm]
                                new_y = np.zeros(y_train.shape)
                                new_y[...] = y_train
                                ind_pos = np.where(y_train == 1)[0]
                                ind_neg = np.where(y_train == 0)[0]

                                # random substitude
                                pca3_cur_pos = v[ind_pos, :]
                                pca3_cur_neg = v[ind_neg, :]
                                to_subst = random.sample(range(num_true), new_true)
                                pca3_pos[to_subst, :] = pca3_cur_pos
                                to_subst = random.sample(range(num_false), new_false)
                                pca3_neg[to_subst, :] = pca3_cur_neg


                    if conf_max < 1 and fail_times >= 2 and update_recent==False:
                        # if id%10==0:
                        update_recent = True
                        pca3_train = np.vstack([pca3_pos, pca3_neg])

                        y_train_true = np.ones((num_true,))
                        y_train_false = np.zeros((num_false,))
                        y_train = np.hstack([y_train_true, y_train_false])

                        # permutation
                        ind_perm = np.random.permutation(range(num_false + num_true))
                        pca3_train = pca3_train[ind_perm, :]

                        y_train = y_train[ind_perm]

                        # logistic regression
                        clf3 = linear_model.LogisticRegression(fit_intercept=True, solver='liblinear')
                        clf3.fit(pca3_train, y_train)

                        # print 'score is: ',clf3.score(pca3_train,y_train)

                # (B,G,R)
                frame_data_cv = frame_data * 255  # [0,1]-->[0,255]
                frame_data_cv = frame_data_cv[:, :, ::-1]  # RGB->BGR
                frame_data_cv = frame_data_cv.astype('uint8')
                cv2.rectangle(frame_data_cv, (int(gt_box[0]), int(gt_box[1])), (int(gt_box[2]), int(gt_box[3])),
                              (255, 0, 0), 2, 1)
                if id > 0 and init_id == True:
                    cv2.rectangle(frame_data_cv, (int(pred_box[0, 0]), int(pred_box[0, 1])),
                                  (int(pred_box[0, 2]), int(pred_box[0, 3])),
                                  (0, 255, 0), 2, 1)
                if init_id == False:
                    init_id = True
                show_particles = False
                if show_particles:
                    for i in range(filter.num_particles):
                        cx = pfilter.particles[i, 0]
                        cy = pfilter.particles[i, 1]
                        cv2.circle(frame_data_cv, (int(cx), int(cy)), 1, (0, 0, 255), thickness=1)
                show_box = False
                if show_box:
                    n = 0
                    for i in ind_pos:
                        if n % 5 == 0:
                            cv2.rectangle(frame_data_cv, (int(boxes_train[i, 0]), int(boxes_train[i, 1])),
                                          (int(boxes_train[i, 2]), int(boxes_train[i, 3])), (0, 0, 255), 2, 1)
                        n += 1
                    n = 0

                show_particles_init = False
                if show_particles_init:
                    for i in range(filter.num_particles):
                        cx = pfilter.particles[i, 0]
                        cy = pfilter.particles[i, 1]
                        cv2.circle(frame_data_cv, (int(cx), int(cy)), 1, (0, 255, 0), thickness=1)
                show_frame = False
                cv2.circle(frame_data_cv, (int(pfilter.cur_c[0]), int(pfilter.cur_c[1])), 2, (0, 0, 255), thickness=1)
                if show_frame:
                    cv2.imshow(sequence, frame_data_cv)
                    c = cv2.waitKey(1)

                    if c != -1:
                        cv2.destroyWindow(sequence)

                        break
                else:
                    video_writer.write(frame_data_cv)
            end_time = time.time()
            video_writer.release()
            print "Average FPS: %f" % (nFrame / (end_time - start_time))
            log_file.write("Average FPS: %f\n" % (nFrame / (end_time - start_time)))
            conf_hist = np.array(conf_hist)
            iou_hist = np.array(iou_hist)
            area_hist = np.array(area_hist)
            pred_hist = np.vstack(pred_hist)
            precisions, auc_pre = utils.calc_prec(gt_boxes, pred_hist)

            suc, auc_iou = utils.calc_success(iou_hist)
            records_precision.append(precisions * nFrame)
            records_success.append(suc * nFrame)

            print 'Precision @20 is: %f' % precisions[19]
            print 'Auc of Precision is: %f' % auc_pre
            print 'Auc of Success is: %f' % auc_iou
            print 'Reinit times: %d' % reinit
            log_file.write("Precision @20 is: %f\n" % precisions[19])
            log_file.write('Auc of Precision is: %f\n' % auc_pre)
            log_file.write('Auc of Success is: %f\n' % auc_iou)
            log_file.write('Reinit times: %d\n' % reinit)
            #log_file.write('Selected feature maps: %d\n' % f_inds.shape[0])
            #log_file.write('PCA components: %d\n' % nPCA)
            #res_f = open('results11/%s.txt'%sequence,'w')
            #pred_hist[:,2:] = pred_hist[:,2:] - pred_hist[:,:2]
            #res_f = write_res(pred_hist,res_f)
            #res_f.close()
    log_file.close()
    pkl = open('results_1031.pkl', 'w')
    pickle.dump([records_precision, records_success], pkl)
    pkl.close()
def main(args):

    vis = args.vis
    debug = args.debug
    save = args.save
    nparticles = args.particles

    root_path = '/home/ccjiang/Documents/caffe-fast-rcnn/examples/tracker/'
    dataset_path = "/data/OTB100"
    sequence = args.sequence
    model_def = os.path.join(root_path, args.prototxt)
    model_weight = os.path.join(root_path, args.caffemodel)
    for t in os.walk(os.path.join(dataset_path, sequence, sequence, "img")):
        if t[0] == os.path.join(dataset_path, sequence, sequence, "img"):
            nFrame = len(t[2])
            print "Total frames are: ", nFrame

    gt_path = os.path.join(dataset_path, sequence, sequence,
                           "groundtruth_rect.txt")

    gt_boxes = utils.get_boxes_all(gt_path)

    vggnet = VGGnet.VGGnet(model_def, model_weight)

    thre_min_neg = 0.0
    thre_max_neg = 0.4  #0.5

    thre_min_pos = 0.8
    thre_max_pos = 1.0

    conf_hist = []
    iou_hist = []
    area_hist = []
    eig_hist = []
    pred_hist = []  #(x1,y1,x2,y2)
    reinit = 0
    nFrame = np.minimum(nFrame, gt_boxes.shape[0])

    for id in np.arange(0, nFrame):  #nFrame
        frame_name = "img/%04d.jpg" % (id + 1)
        print "Start processing: %s" % frame_name
        frame_path = os.path.join(dataset_path, sequence, sequence, frame_name)
        frame_data = caffe.io.load_image(frame_path)  # (432,576,3), in [0,1]
        gt_box = gt_boxes[id]

        if id == 0:
            h, w, c = frame_data.shape
            frame_shape = [c, w, h]
            fps = 20
            fourcc = cv2.VideoWriter_fourcc(*'MJPG')

            video_writer = cv2.VideoWriter("res_%s.avi" % sequence, fourcc,
                                           fps, (w, h))
            fail_times = 0
            area = (gt_box[2] - gt_box[0]) * (gt_box[3] - gt_box[1])
            ratio = (gt_box[2] - gt_box[0]) / (gt_box[3] - gt_box[1]
                                               )  #ratio=w/h
            # set up net.blobs['im_info']
            print "Image Size: ", w, h
            vggnet.reshape(w=w, h=h, nbox=nparticles)
            filter = PFfilter.PFfilter(
                utils.bbox_to_states(gt_box, area, ratio), area, ratio, w, h,
                nparticles)
            filter.create_particles()
            filter.restrict_particles(w, h)
            area_hist.append(filter.cur_a)
            pred_hist.append(np.array(gt_box).reshape(1, -1))
            #pca
            # test sample_iou
            num_true = 500
            num_false = 1000
            boxes_train = []
            #boxes_train_neg=[]
            iou_train = []
            try:
                #Q=[[1,0],[0,1]] #for pixel wise
                Q = 0.05  #box_w,box_h
                sample_box_true, sample_iou_true = filter.sample_iou(
                    gt_box, Q, 0.01, 0.01, num_true, 0.8, 1.0)
            except OverflowError as e:
                print "too many loops in sample."
            # print sample_box_true[:10]
            # print sample_box_true.shape[0]
            # print sample_iou_true[:10]
            print "average iou: ", np.mean(sample_iou_true)
            boxes_train.append(sample_box_true)
            iou_train.append(sample_iou_true)
            try:
                #Q=[[36,0],[0,36]]#for pixel wise
                Q = 0.2  #0.15
                sample_box_false, sample_iou_false = filter.sample_iou(
                    gt_box, Q, 0.2, 0.01, num_false / 2, 0, thre_max_neg)
            except OverflowError as e:
                print "too many loops in sample."
            # print sample_box_false[:10]
            # print sample_box_false.shape[0]
            # print sample_iou_false[:10]
            print "average iou: ", np.mean(sample_iou_false)
            boxes_train.append(sample_box_false)
            iou_train.append(sample_iou_false)
            try:
                #Q=[[36,0],[0,36]]#for pixel wise
                Q = 0.2
                sample_box_false, sample_iou_false = filter.sample_iou(
                    gt_box, Q, 0.01, 0.2, num_false / 2, 0, thre_max_neg)
            except OverflowError as e:
                print "too many loops in sample."
            # print sample_box_false[:10]
            # print sample_box_false.shape[0]
            # print sample_iou_false[:10]
            print "average iou: ", np.mean(sample_iou_false)
            boxes_train.append(sample_box_false)
            iou_train.append(sample_iou_false)

            boxes_train = np.vstack(boxes_train)

            iou_train = np.vstack(iou_train)
            y_train_true = np.ones((num_true, ))
            y_train_false = np.zeros((num_false, ))

            y_train = np.hstack([y_train_true, y_train_false])

            #permutation
            ind_perm = np.random.permutation(range(num_false + num_true))
            boxes_train = boxes_train[ind_perm, :]
            iou_train = iou_train[ind_perm]
            y_train = y_train[ind_perm]
            ind_pos = np.where(y_train == 1)[0]
            ind_neg = np.where(y_train == 0)[0]

            vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0])
            features = vggnet.get_features_first_raw(frame_data,
                                                     boxes_raw=boxes_train,
                                                     id=id)

            for k, v in features.iteritems():
                print k, v.shape
                if k == 'f3':
                    pca3 = utils.skl_pca(v)
                    v_pca3 = pca3.transform(v)
                    pca3_pos = np.zeros((num_true, pca3.n_components_),
                                        dtype=np.float32)
                    pca3_neg = np.zeros((num_false, pca3.n_components_),
                                        dtype=np.float32)
                    pca3_pos[...] = v_pca3[ind_pos, :]
                    pca3_neg[...] = v_pca3[ind_neg, :]
                    #utils.vis_as_image(v_pca3)
                    #plt.imshow(v_pca3)
                    #plt.title("PCA features")
                    #plt.show()
                    #plt.close()
                    #logistic regression
                    y_weight = sklearn.utils.class_weight.compute_class_weight(
                        class_weight='balanced',
                        classes=np.array([0, 1]),
                        y=y_train)
                    #print y_weight
                    class_weight = {0: y_weight[0], 1: y_weight[1]}
                    clf3 = SVC(kernel="linear")
                    #clf3=linear_model.LogisticRegression(fit_intercept=True,solver='liblinear')
                    clf3.fit(v_pca3, y_train)
                    score3 = clf3.score(v_pca3, y_train)
                    print 'score3: ', score3
                    #prob=clf3.predict_proba(v_pca3)
                    print clf3.classes_
                    #print prob

            vis_feature = False
            if vis_feature:
                utils.vis_features(features, id)

            start_time = time.time()
        else:
            if fail_times >= 5:
                #reinitialize
                reinit += 1
                area = (gt_box[2] - gt_box[0]) * (gt_box[3] - gt_box[1])
                ratio = (gt_box[2] - gt_box[0]) / (gt_box[3] - gt_box[1])
                filter = PFfilter.PFfilter(
                    utils.bbox_to_states(gt_box, area, ratio), area, ratio, w,
                    h, nparticles)
                #filter.reset(utils.bbox_to_states(gt_box, area, ratio), area, ratio)
                filter.create_particles()
                filter.restrict_particles(w, h)
                area_hist.append(filter.cur_a)
                pred_box = gt_box
                boxes_train = []
                pred_hist.append(np.array(gt_box).reshape(1, -1))
                #pred_hist.append(pred_box)
                conf_hist.append(-0.1)

                # boxes_train_neg=[]
                iou_train = []
                try:
                    # Q=[[1,0],[0,1]] #for pixel wise
                    Q = 0.05  # box_w,box_h
                    sample_box_true, sample_iou_true = filter.sample_iou(
                        gt_box, Q, 0.01, 0.01, num_true, 0.8, 1.0)
                except OverflowError as e:
                    print "too many loops in sample."
                # print sample_box_true[:10]
                # print sample_box_true.shape[0]
                # print sample_iou_true[:10]
                print "average iou: ", np.mean(sample_iou_true)
                boxes_train.append(sample_box_true)
                iou_train.append(sample_iou_true)
                try:
                    # Q=[[36,0],[0,36]]#for pixel wise
                    Q = 0.2  #0.15
                    sample_box_false, sample_iou_false = filter.sample_iou(
                        gt_box, Q, 0.2, 0.01, num_false / 2, 0, thre_max_neg)
                except OverflowError as e:
                    print "too many loops in sample."
                # print sample_box_false[:10]
                # print sample_box_false.shape[0]
                # print sample_iou_false[:10]
                print "average iou: ", np.mean(sample_iou_false)
                boxes_train.append(sample_box_false)
                iou_train.append(sample_iou_false)
                try:
                    # Q=[[36,0],[0,36]]#for pixel wise
                    Q = 0.2
                    sample_box_false, sample_iou_false = filter.sample_iou(
                        gt_box, Q, 0.01, 0.2, num_false / 2, 0, thre_max_neg)
                except OverflowError as e:
                    print "too many loops in sample."
                # print sample_box_false[:10]
                # print sample_box_false.shape[0]
                # print sample_iou_false[:10]
                print "average iou: ", np.mean(sample_iou_false)
                boxes_train.append(sample_box_false)
                iou_train.append(sample_iou_false)

                boxes_train = np.vstack(boxes_train)
                iou_train = np.vstack(iou_train)
                y_train_true = np.ones((num_true, ))
                y_train_false = np.zeros((num_false, ))

                y_train = np.hstack([y_train_true, y_train_false])
                # permutation
                ind_perm = np.random.permutation(range(num_false + num_true))
                boxes_train = boxes_train[ind_perm, :]
                iou_train = iou_train[ind_perm]
                y_train = y_train[ind_perm]
                ind_pos = np.where(y_train == 1)[0]
                ind_neg = np.where(y_train == 0)[0]

                vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0])
                features = vggnet.get_features_first_raw(frame_data,
                                                         boxes_raw=boxes_train,
                                                         id=id)

                for k, v in features.iteritems():
                    print k, v.shape
                    if k == 'f3':

                        v_pca3 = pca3.transform(v)

                        pca3_pos[...] = v_pca3[ind_pos, :]
                        pca3_neg[...] = v_pca3[ind_neg, :]
                        clf3.fit(v_pca3, y_train)
                        score3 = clf3.score(v_pca3, y_train)
                        print 'score3: ', score3
                        # prob=clf3.predict_proba(v_pca3)
                        print clf3.classes_
                fail_times = 0
                continue

            filter.predict_particles(Q=0.02, cr=0.05,
                                     ca=0.05)  #0.02,0.0005,0.005
            filter.restrict_particles(w, h)
            area_hist.append(filter.cur_a)
            #compute conf
            conf = np.zeros(filter.weights.shape)
            #np.save('particles.npy',filter.particles)
            pred_boxes = utils.state_to_bbox(filter.particles, area, ratio)
            vggnet.reshape(w, h, filter.num_particles)
            features = vggnet.get_features_first_raw(frame_data,
                                                     boxes_raw=pred_boxes,
                                                     id=id)
            for k, v in features.iteritems():
                print k, v.shape
                if k == 'f3':

                    v_pca3 = pca3.transform(v)
                    #utils.vis_as_image(v_pca3)
                    #plt.imshow(v_pca3)
                    #plt.title("PCA features")
                    #plt.show()
                    #plt.close()
                    #logistic regression
                    #conf=clf3.predict_proba(v_pca3)[:,1]
                    conf = -clf3.decision_function(v_pca3)

            conf_max = np.max(conf)
            conf_min = np.min(conf)
            print 'conf_max: ', conf_max
            print 'conf_min: ', conf_min
            filter.update_particles(conf)
            # pred_state = filter.estimate()
            print filter.weights
            filter.resample()  # always resample
            pred_state, s_particles, r_particles = filter.estimate(k=10)

            cov_particles = np.dot(
                filter.particles[:, :4].T,
                filter.particles[:, :4]) / filter.particles.shape[0]

            eigval, eigvec = np.linalg.eig(cov_particles)
            max_val = eigval[0]
            eig_hist.append(max_val)
            print 'Max eigvalue: %f' % max_val

            #print 'conf is: ',conf
            if conf_max > 0 and max_val < 200000:
                fail_times = 0

            else:
                fail_times += 1
                #filter.update_particles(conf)
                #pred_state=filter.estimate()
                #filter.resample()
                #pred_state, s_particles, r_particles = filter.estimate(k=10)
                print "conf_max too low, not update particles "
            pred_box = utils.state_to_bbox(pred_state.reshape((-1, 6)), area,
                                           ratio)
            print 'ground truth bbox is: ', gt_box
            print "pred_box is: ", pred_box
            show_sr = False
            if show_sr:
                plt.hist2d(s_particles,
                           r_particles,
                           bins=50,
                           weights=filter.weights.squeeze())
                '''
                plt.scatter(s_particles,r_particles,c='r',marker='.',linewidths=1)
                plt.xlabel('Area')
                plt.ylabel('Aspect ratio')
                plt.title('Area and Ratio of particles')
                plt.axis('equal')
                '''
                plt.show()
            iou = utils.calc_iou(gt_box, pred_box)
            print 'iou is: ', iou
            pred_hist.append(pred_box)
            conf_hist.append(conf_max)
            iou_hist.append(iou)

            if conf_max >= 0.1:  #0.5
                #update pca3_pos and pca3_neg
                new_true = 100  #50
                new_false = 200  #100
                boxes_train = []

                iou_train = []
                Q = 0.02
                try:
                    sample_box_true, sample_iou_true = filter.sample_iou(
                        pred_box, Q, 0.01, 0.01, new_true, 0.85, 1.0)
                except OverflowError as e:
                    print "too many loops in sample."
                # print sample_box_true[:10]
                # print sample_box_true.shape[0]
                # print sample_iou_true[:10]
                print "average iou: ", np.mean(sample_iou_true)
                boxes_train.append(sample_box_true)
                iou_train.append(sample_iou_true)
                try:
                    Q = 0.2
                    sample_box_false, sample_iou_false = filter.sample_iou(
                        pred_box, Q, 0.2, 0.01, new_false / 2, 0, thre_max_neg)
                except OverflowError as e:
                    print "too many loops in sample."
                # print sample_box_false[:10]
                # print sample_box_false.shape[0]
                # print sample_iou_false[:10]
                print "average iou: ", np.mean(sample_iou_false)
                boxes_train.append(sample_box_false)
                iou_train.append(sample_iou_false)
                try:
                    Q = 0.2
                    sample_box_false, sample_iou_false = filter.sample_iou(
                        pred_box, Q, 0.01, 0.2, new_false / 2, 0, thre_max_neg)
                except OverflowError as e:
                    print "too many loops in sample."
                # print sample_box_false[:10]
                # print sample_box_false.shape[0]
                # print sample_iou_false[:10]
                print "average iou: ", np.mean(sample_iou_false)
                boxes_train.append(sample_box_false)
                iou_train.append(sample_iou_false)

                boxes_train = np.vstack(boxes_train)

                iou_train = np.vstack(iou_train)
                y_train_true = np.ones((new_true, ))
                y_train_false = np.zeros((new_false, ))
                y_train = np.hstack([y_train_true, y_train_false])

                # permutation
                ind_perm = np.random.permutation(range(new_false + new_true))
                boxes_train = boxes_train[ind_perm, :]

                y_train = y_train[ind_perm]
                new_y = np.zeros(y_train.shape)
                new_y[...] = y_train
                ind_pos = np.where(y_train == 1)[0]
                ind_neg = np.where(y_train == 0)[0]

                vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0])
                features = vggnet.get_features_first_raw(frame_data,
                                                         boxes_raw=boxes_train,
                                                         id=id)
                for k, v in features.iteritems():
                    print k, v.shape
                    if k == 'f3':
                        v_pca3 = pca3.transform(v)

                        #random substitude
                        pca3_cur_pos = v_pca3[ind_pos, :]
                        pca3_cur_neg = v_pca3[ind_neg, :]
                        to_subst = random.sample(range(num_true), new_true)
                        pca3_pos[to_subst, :] = pca3_cur_pos
                        to_subst = random.sample(range(num_false), new_false)
                        pca3_neg[to_subst, :] = pca3_cur_neg

            if conf_max < 0.1 and fail_times >= 2:  #0.99
                #if conf_max<0.95 and conf_max>0.5:
                #update classification model
                print 'updating model...'
                pca3_train = np.vstack([pca3_pos, pca3_neg])

                y_train_true = np.ones((num_true, ))
                y_train_false = np.zeros((num_false, ))
                y_train = np.hstack([y_train_true, y_train_false])

                # permutation
                ind_perm = np.random.permutation(range(num_false + num_true))
                pca3_train = pca3_train[ind_perm, :]

                y_train = y_train[ind_perm]

                #logistic regression

                clf3.fit(pca3_train, y_train)
                print 'score is: ', clf3.score(pca3_train, y_train)
                #fail_times=0

        # (B,G,R)
        frame_data_cv = frame_data * 255  # [0,1]-->[0,255]
        frame_data_cv = frame_data_cv[:, :, ::-1]  # RGB->BGR
        frame_data_cv = frame_data_cv.astype('uint8')
        #cv2.rectangle(frame_data_cv, (int(gt_box[0]), int(gt_box[1])), (int(gt_box[2]), int(gt_box[3])),
        #                         (255, 0, 0), 2, 1)
        if id > 0:
            cv2.rectangle(frame_data_cv,
                          (int(pred_box[0, 0]), int(pred_box[0, 1])),
                          (int(pred_box[0, 2]), int(pred_box[0, 3])),
                          (0, 255, 0), 2, 1)
        show_particles = False
        if show_particles:
            for i in range(filter.num_particles):
                cx = filter.particles[i, 0]
                cy = filter.particles[i, 1]
                cv2.circle(frame_data_cv, (int(cx), int(cy)),
                           1, (0, 0, 255),
                           thickness=1)
        show_box = False
        if show_box:
            n = 0
            for i in ind_pos:
                if n % 5 == 0:
                    cv2.rectangle(
                        frame_data_cv,
                        (int(boxes_train[i, 0]), int(boxes_train[i, 1])),
                        (int(boxes_train[i, 2]), int(boxes_train[i, 3])),
                        (0, 0, 255), 2, 1)
                n += 1
            n = 0

        show_particles_init = False
        if show_particles_init:
            for i in range(filter.num_particles):
                cx = filter.particles[i, 0]
                cy = filter.particles[i, 1]
                cv2.circle(frame_data_cv, (int(cx), int(cy)),
                           1, (0, 255, 0),
                           thickness=1)
        show_frame = False
        #cv2.circle(frame_data_cv, (int(filter.cur_c[0]), int(filter.cur_c[1])), 2, (0, 0, 255), thickness=1)

        if show_frame:
            cv2.imshow(sequence, frame_data_cv)

            c = cv2.waitKey(1)
            if c != -1:
                if chr(c) == 'p':
                    c = cv2.waitKey()
                #print 'You press: ',chr(c)
                #if chr(c)=='c':
                if chr(c) == 'c':
                    cv2.destroyWindow(sequence)
                    #conf_hist=np.array(conf_hist)
                    #iou_hist=np.array(iou_hist)
                    #np.save('conf_hist.npy',conf_hist)
                    #np.save('iou_hist.npy',iou_hist)
                    break
        else:
            video_writer.write(frame_data_cv)
    end_time = time.time()
    video_writer.release()
    iou_hist = np.array(iou_hist)
    pred_hist = np.array(pred_hist).squeeze()
    print "iou_hist: ", iou_hist.shape
    print "pred_hist: ", pred_hist.shape
    print "get_boxes: ", gt_boxes.shape
    precisions, auc_pre = utils.calc_prec(gt_boxes, pred_hist)
    print "precision is: %f" % (precisions[19])
    suc, auc_iou = utils.calc_success(iou_hist)
    print "Average IOU is: %f" % (np.mean(iou_hist))
    print "Auc of precision is: %f" % (auc_pre)
    print "Auc of success is: %f" % auc_iou
    print "Reinit times: %d" % reinit
    print "Average FPS: %f" % ((id + 1) / (end_time - start_time))
def main(args):
    vis = args.vis
    debug = args.debug
    save = args.save
    nparticles = args.particles
    root_path = '/home/ccjiang/Documents/caffe-fast-rcnn/examples/tracker/'
    dataset_path = args.dataset  #"/data/OTB100"
    dataset100_seq = [
        'Bird2', 'BlurCar1', 'BlurCar3', 'BlurCar4', 'Board', 'Bolt2', 'Boy',
        'Car2', 'Car24', 'Coke', 'Coupon', 'Crossing', 'Dancer', 'Dancer2',
        'David2', 'David3', 'Dog', 'Dog1', 'Doll', 'FaceOcc1', 'FaceOcc2',
        'Fish', 'FleetFace', 'Football1', 'Freeman1', 'Freeman3', 'Girl2',
        'Gym', 'Human2', 'Human5', 'Human7', 'Human8', 'Jogging', 'KiteSurf',
        'Lemming', 'Man', 'Mhyang', 'MountainBike', 'Rubik', 'Singer1',
        'Skater', 'Skater2', 'Subway', 'Suv', 'Tiger1', 'Toy', 'Trans',
        'Twinnings', 'Vase'
    ]
    dataset50_seq = [
        'Basketball', 'Biker', 'Bird1', 'BlurBody', 'BlurCar2', 'BlurFace',
        'BlurOwl', 'Bolt', 'Box', 'Car1', 'Car4', 'CarDark', 'CarScale',
        'ClifBar', 'Couple', 'Crowds', 'Deer', 'Diving', 'DragonBaby', 'Dudek',
        'Football', 'Freeman4', 'Girl', 'Human3', 'Human4', 'Human6', 'Human9',
        'Ironman', 'Jump', 'Jumping', 'Liquor', 'Matrix', 'MotorRolling',
        'Panda', 'RedTeam', 'Shaking', 'Singer2', 'Skating1', 'Skating2',
        'Skiing', 'Soccer', 'Surfer', 'Sylvester', 'Tiger2', 'Trellis',
        'Walking', 'Walking2', 'Woman'
    ]
    if "OTB50" in dataset_path:
        data_seq = dataset50_seq
    else:
        data_seq = dataset100_seq

    log_name = 'log.txt'
    log_file = open(log_name, 'w')
    records_success = []  #defaultdict(list)
    records_precision = []  #defaultdict(list)
    records_reinit = defaultdict(list)
    model_def = os.path.join(root_path, args.prototxt)
    model_weight = os.path.join(root_path, args.caffemodel)
    vggnet = VGGnet.VGGnet(model_def, model_weight)

    thre_max_neg = 0.3  # 0.5
    test_times = 1  # 0
    for t in range(test_times):
        print 'Test round: %d' % t
        log_file.write('Test round: %d\n' % t)
        # sequences = ['Fish']
        for sequence in data_seq:
            for t in os.walk(
                    os.path.join(dataset_path, sequence, sequence, "img")):
                if t[0] == os.path.join(dataset_path, sequence, sequence,
                                        "img"):
                    nFrame = len(t[2])
                    print 'Processing: %s' % sequence
                    log_file.write('Processing: %s\n' % sequence)
                    print "Total frames are: ", nFrame
                    log_file.write('Total frames are: %d\n' % nFrame)
            gt_path = os.path.join(dataset_path, sequence, sequence,
                                   "groundtruth_rect.txt")

            gt_boxes = utils.get_boxes_all(gt_path)

            conf_hist = []
            iou_hist = []
            area_hist = []
            pred_hist = []  # (x1,y1,x2,y2)
            eig_hist = []
            reinit = 0
            nFrame = np.minimum(nFrame, gt_boxes.shape[0])
            id_shift = 0
            init_id = False
            for id in np.arange(0, nFrame):
                frame_name = "img/%04d.jpg" % (id + 1)
                # print "Start processing: %s" % frame_name
                frame_path = os.path.join(dataset_path, sequence, sequence,
                                          frame_name)
                if os.path.exists(frame_path) == False:
                    id_shift = id_shift + 1
                    continue
                id = id - id_shift

                frame_data = caffe.io.load_image(
                    frame_path)  # (432,576,3), in [0,1]
                gt_box = gt_boxes[id]

                if init_id == False:
                    h, w, c = frame_data.shape
                    frame_shape = [c, w, h]

                    fail_times = 0
                    area = (gt_box[2] - gt_box[0]) * (gt_box[3] - gt_box[1])
                    ratio = (gt_box[2] - gt_box[0]) / (gt_box[3] - gt_box[1]
                                                       )  # ratio=w/h
                    # set up net.blobs['im_info']
                    print "Image Size: ", w, h
                    log_file.write('Image Size: %d %d\n' % (w, h))
                    vggnet.reshape(w=w, h=h, nbox=nparticles)
                    filter = PFfilter.PFfilter(
                        utils.bbox_to_states(gt_box, area, ratio), area, ratio,
                        w, h, nparticles)
                    filter.create_particles()
                    filter.restrict_particles(w, h)
                    area_hist.append(filter.cur_a)
                    pred_hist.append(np.array(gt_box).reshape(1, -1))
                    # pca
                    # test sample_iou
                    num_true = 500
                    num_false = 1000
                    boxes_train = []
                    # boxes_train_neg=[]
                    iou_train = []
                    try:
                        # Q=[[1,0],[0,1]] #for pixel wise
                        Q = 0.05  # box_w,box_h
                        sample_box_true, sample_iou_true = filter.sample_iou_pred_box(
                            gt_box, Q, 0.01, 0.01, num_true, 0.8, 1.0)
                    except OverflowError as e:
                        print "too many loops in sample."
                    # print sample_box_true[:10]
                    # print sample_box_true.shape[0]
                    # print sample_iou_true[:10]
                    # print "average iou: ", np.mean(sample_iou_true)
                    boxes_train.append(sample_box_true)
                    iou_train.append(sample_iou_true)
                    try:
                        # Q=[[36,0],[0,36]]#for pixel wise
                        Q = 0.2  # 0.15
                        sample_box_false, sample_iou_false = filter.sample_iou(
                            gt_box, Q, 0.2, 0.01, num_false / 2, 0,
                            thre_max_neg)
                    except OverflowError as e:
                        print "too many loops in sample."
                    # print sample_box_false[:10]
                    # print sample_box_false.shape[0]
                    # print sample_iou_false[:10]
                    # print "average iou: ", np.mean(sample_iou_false)
                    boxes_train.append(sample_box_false)
                    iou_train.append(sample_iou_false)
                    try:
                        # Q=[[36,0],[0,36]]#for pixel wise
                        Q = 0.2
                        sample_box_false, sample_iou_false = filter.sample_iou(
                            gt_box, Q, 0.01, 0.2, num_false / 2, 0,
                            thre_max_neg)
                    except OverflowError as e:
                        print "too many loops in sample."
                    # print sample_box_false[:10]
                    # print sample_box_false.shape[0]
                    # print sample_iou_false[:10]
                    # print "average iou: ", np.mean(sample_iou_false)
                    boxes_train.append(sample_box_false)
                    iou_train.append(sample_iou_false)

                    boxes_train = np.vstack(boxes_train)

                    iou_train = np.vstack(iou_train)
                    y_train_true = np.ones((num_true, ))
                    y_train_false = np.zeros((num_false, ))

                    y_train = np.hstack([y_train_true, y_train_false])

                    # permutation
                    ind_perm = np.random.permutation(
                        range(num_false + num_true))
                    boxes_train = boxes_train[ind_perm, :]
                    iou_train = iou_train[ind_perm]
                    y_train = y_train[ind_perm]
                    ind_pos = np.where(y_train == 1)[0]
                    ind_neg = np.where(y_train == 0)[0]

                    vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0])
                    features = vggnet.get_features_first_raw(
                        frame_data, boxes_raw=boxes_train, id=id)

                    for k, v in features.iteritems():
                        # print k,v.shape
                        if k == 'f3':
                            pca3 = utils.skl_pca(v)
                            v_pca3 = pca3.transform(v)
                            pca3_pos = np.zeros((num_true, pca3.n_components_),
                                                dtype=np.float32)
                            pca3_neg = np.zeros(
                                (num_false, pca3.n_components_),
                                dtype=np.float32)
                            pca3_pos[...] = v_pca3[ind_pos, :]
                            pca3_neg[...] = v_pca3[ind_neg, :]
                            # utils.vis_as_image(v_pca3)
                            # plt.imshow(v_pca3)
                            # plt.title("PCA features")
                            # plt.show()
                            # plt.close()
                            # logistic regression
                            y_weight = sklearn.utils.class_weight.compute_class_weight(
                                class_weight='balanced',
                                classes=np.array([0, 1]),
                                y=y_train)
                            # print y_weight
                            class_weight = {0: y_weight[0], 1: y_weight[1]}
                            clf3 = linear_model.LogisticRegression(
                                fit_intercept=True, solver='liblinear')
                            clf3.fit(v_pca3, y_train)

                    vis_feature = False
                    if vis_feature:
                        utils.vis_features(features, id)

                    start_time = time.time()
                else:
                    if fail_times >= 5:
                        # reinitialize
                        reinit += 1
                        area = (gt_box[2] - gt_box[0]) * (gt_box[3] -
                                                          gt_box[1])
                        ratio = (gt_box[2] - gt_box[0]) / (gt_box[3] -
                                                           gt_box[1])
                        filter = PFfilter.PFfilter(
                            utils.bbox_to_states(gt_box, area, ratio), area,
                            ratio, w, h, nparticles)
                        # filter.reset(utils.bbox_to_states(gt_box, area, ratio), area, ratio)
                        filter.create_particles()
                        filter.restrict_particles(w, h)
                        area_hist.append(filter.cur_a)
                        pred_box = gt_box
                        pred_hist.append(np.array(pred_box).reshape(1, -1))
                        conf_hist.append(-0.1)
                        boxes_train = []
                        # boxes_train_neg=[]
                        iou_train = []
                        try:
                            # Q=[[1,0],[0,1]] #for pixel wise
                            Q = 0.05  # box_w,box_h
                            sample_box_true, sample_iou_true = filter.sample_iou_pred_box(
                                gt_box, Q, 0.01, 0.01, num_true, 0.8, 1.0)
                        except OverflowError as e:
                            print "too many loops in sample."

                        boxes_train.append(sample_box_true)
                        iou_train.append(sample_iou_true)
                        try:
                            # Q=[[36,0],[0,36]]#for pixel wise
                            Q = 0.2  # 0.15
                            sample_box_false, sample_iou_false = filter.sample_iou(
                                gt_box, Q, 0.2, 0.01, num_false / 2, 0,
                                thre_max_neg)
                        except OverflowError as e:
                            print "too many loops in sample."

                        boxes_train.append(sample_box_false)
                        iou_train.append(sample_iou_false)
                        try:
                            # Q=[[36,0],[0,36]]#for pixel wise
                            Q = 0.2
                            sample_box_false, sample_iou_false = filter.sample_iou(
                                gt_box, Q, 0.01, 0.2, num_false / 2, 0,
                                thre_max_neg)
                        except OverflowError as e:
                            print "too many loops in sample."
                        # print sample_box_false[:10]
                        # print sample_box_false.shape[0]
                        # print sample_iou_false[:10]
                        # print "average iou: ", np.mean(sample_iou_false)
                        boxes_train.append(sample_box_false)
                        iou_train.append(sample_iou_false)

                        boxes_train = np.vstack(boxes_train)
                        iou_train = np.vstack(iou_train)
                        y_train_true = np.ones((num_true, ))
                        y_train_false = np.zeros((num_false, ))

                        y_train = np.hstack([y_train_true, y_train_false])
                        # permutation
                        ind_perm = np.random.permutation(
                            range(num_false + num_true))
                        boxes_train = boxes_train[ind_perm, :]
                        iou_train = iou_train[ind_perm]
                        y_train = y_train[ind_perm]
                        ind_pos = np.where(y_train == 1)[0]
                        ind_neg = np.where(y_train == 0)[0]

                        vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0])
                        features = vggnet.get_features_first_raw(
                            frame_data, boxes_raw=boxes_train, id=id)

                        for k, v in features.iteritems():
                            # print k, v.shape
                            if k == 'f3':
                                v_pca3 = pca3.transform(v)

                                pca3_pos[...] = v_pca3[ind_pos, :]
                                pca3_neg[...] = v_pca3[ind_neg, :]
                                clf3.fit(v_pca3, y_train)
                                score3 = clf3.score(v_pca3, y_train)
                                # print 'score3: ', score3
                                # prob=clf3.predict_proba(v_pca3)
                                # print clf3.classes_
                        fail_times = 0
                        continue

                    filter.predict_particles(Q=0.2, cr=0.005, ca=0.001)
                    filter.restrict_particles(w, h)
                    area_hist.append(filter.cur_a)
                    # compute conf
                    conf = np.zeros(filter.weights.shape)
                    # np.save('particles.npy',filter.particles)
                    pred_boxes = utils.state_to_bbox(filter.particles, area,
                                                     ratio)
                    vggnet.reshape(w, h, filter.num_particles)
                    features = vggnet.get_features_first_raw(
                        frame_data, boxes_raw=pred_boxes, id=id)
                    for k, v in features.iteritems():
                        # print k,v.shape
                        if k == 'f3':
                            v_pca3 = pca3.transform(v)
                            # utils.vis_as_image(v_pca3)
                            # plt.imshow(v_pca3)
                            # plt.title("PCA features")
                            # plt.show()
                            # plt.close()
                            # logistic regression
                            conf = clf3.predict_proba(v_pca3)[:, 1]

                    conf_max = np.max(conf)
                    conf_min = np.min(conf)
                    filter.update_particles(conf)
                    filter.resample()  # always resample
                    pred_state, s_particles, r_particles = filter.estimate(
                        k=10)

                    cov_particles = np.dot(
                        filter.particles[:, :2].T,
                        filter.particles[:, :2]) / filter.particles.shape[0]

                    eigval, eigvec = np.linalg.eig(cov_particles)
                    max_val = eigval[0]
                    eig_hist.append(max_val)
                    print 'Max eigvalue: %f' % max_val

                    # print 'conf is: ',conf
                    if conf_max > 0.8:
                        fail_times = 0

                    else:
                        fail_times += 1

                        print "conf_max too low, not update particles "
                    pred_box = utils.state_to_bbox(pred_state.reshape((-1, 6)),
                                                   area, ratio)
                    show_sr = False
                    if show_sr:
                        count, xedge, yedge, tmp_im = plt.hist2d(
                            s_particles,
                            r_particles,
                            bins=10,
                            weights=filter.weights.squeeze(),
                            cmap=plt.cm.gray)
                        top3 = np.argsort(-count, axis=None)[:3]
                        row_ind = top3[:] / count.shape[1]
                        col_ind = top3[:] % count.shape[0]
                        '''
                        plt.scatter(s_particles,r_particles,c='r',marker='.',linewidths=1)
                        plt.xlabel('Area')
                        plt.ylabel('Aspect ratio')
                        plt.title('Area and Ratio of particles')
                        plt.axis('equal')
                        '''
                        plt.show()
                    iou = utils.calc_iou(gt_box, pred_box)
                    # print 'iou is: ', iou
                    pred_hist.append(pred_box)
                    conf_hist.append(conf_max)
                    iou_hist.append(iou)

                    if conf_max >= 0.9:  # 0.8
                        # update pca3_pos and pca3_neg
                        new_true = 100  # 50
                        new_false = 200  # 100
                        boxes_train = []

                        iou_train = []
                        Q = 0.02
                        try:
                            sample_box_true, sample_iou_true = filter.sample_iou_pred_box(
                                pred_box, Q, 0.01, 0.01, new_true, 0.85, 1.0)
                        except OverflowError as e:
                            print "too many loops in sample."
                        # print sample_box_true[:10]
                        # print sample_box_true.shape[0]
                        # print sample_iou_true[:10]
                        # print "average iou: ", np.mean(sample_iou_true)
                        boxes_train.append(sample_box_true)
                        iou_train.append(sample_iou_true)
                        try:
                            Q = 0.2
                            sample_box_false, sample_iou_false = filter.sample_iou(
                                pred_box, Q, 0.2, 0.01, new_false / 2, 0,
                                thre_max_neg)
                        except OverflowError as e:
                            print "too many loops in sample."
                        # print sample_box_false[:10]
                        # print sample_box_false.shape[0]
                        # print sample_iou_false[:10]
                        # print "average iou: ", np.mean(sample_iou_false)
                        boxes_train.append(sample_box_false)
                        iou_train.append(sample_iou_false)
                        try:
                            Q = 0.2
                            sample_box_false, sample_iou_false = filter.sample_iou(
                                pred_box, Q, 0.01, 0.2, new_false / 2, 0,
                                thre_max_neg)
                        except OverflowError as e:
                            print "too many loops in sample."
                        boxes_train.append(sample_box_false)
                        iou_train.append(sample_iou_false)

                        boxes_train = np.vstack(boxes_train)

                        iou_train = np.vstack(iou_train)
                        y_train_true = np.ones((new_true, ))
                        y_train_false = np.zeros((new_false, ))
                        y_train = np.hstack([y_train_true, y_train_false])

                        # permutation
                        ind_perm = np.random.permutation(
                            range(new_false + new_true))
                        boxes_train = boxes_train[ind_perm, :]

                        y_train = y_train[ind_perm]
                        new_y = np.zeros(y_train.shape)
                        new_y[...] = y_train
                        ind_pos = np.where(y_train == 1)[0]
                        ind_neg = np.where(y_train == 0)[0]

                        vggnet.reshape(w=w, h=h, nbox=boxes_train.shape[0])
                        features = vggnet.get_features_second_raw(
                            boxes_raw=boxes_train, id=id)
                        for k, v in features.iteritems():
                            # print k, v.shape
                            if k == 'f3':
                                v_pca3 = pca3.transform(v)

                                # random substitude
                                pca3_cur_pos = v_pca3[ind_pos, :]
                                pca3_cur_neg = v_pca3[ind_neg, :]
                                to_subst = random.sample(
                                    range(num_true), new_true)
                                pca3_pos[to_subst, :] = pca3_cur_pos
                                to_subst = random.sample(
                                    range(num_false), new_false)
                                pca3_neg[to_subst, :] = pca3_cur_neg

                    if conf_max < 1 and fail_times >= 2:
                        pca3_train = np.vstack([pca3_pos, pca3_neg])

                        y_train_true = np.ones((num_true, ))
                        y_train_false = np.zeros((num_false, ))
                        y_train = np.hstack([y_train_true, y_train_false])

                        # permutation
                        ind_perm = np.random.permutation(
                            range(num_false + num_true))
                        pca3_train = pca3_train[ind_perm, :]

                        y_train = y_train[ind_perm]

                        # logistic regression

                        clf3.fit(pca3_train, y_train)
                        # print 'score is: ',clf3.score(pca3_train,y_train)

                # (B,G,R)
                frame_data_cv = frame_data * 255  # [0,1]-->[0,255]
                frame_data_cv = frame_data_cv[:, :, ::-1]  # RGB->BGR
                frame_data_cv = frame_data_cv.astype('uint8')
                cv2.rectangle(frame_data_cv, (int(gt_box[0]), int(gt_box[1])),
                              (int(gt_box[2]), int(gt_box[3])), (255, 0, 0), 2,
                              1)
                if id > 0 and init_id == True:
                    cv2.rectangle(frame_data_cv,
                                  (int(pred_box[0, 0]), int(pred_box[0, 1])),
                                  (int(pred_box[0, 2]), int(pred_box[0, 3])),
                                  (0, 255, 0), 2, 1)
                if init_id == False:
                    init_id = True
                show_particles = False
                if show_particles:
                    for i in range(filter.num_particles):
                        cx = filter.particles[i, 0]
                        cy = filter.particles[i, 1]
                        cv2.circle(frame_data_cv, (int(cx), int(cy)),
                                   1, (0, 0, 255),
                                   thickness=1)
                show_box = False
                if show_box:
                    n = 0
                    for i in ind_pos:
                        if n % 5 == 0:
                            cv2.rectangle(frame_data_cv, (int(
                                boxes_train[i, 0]), int(boxes_train[i, 1])),
                                          (int(boxes_train[i, 2]),
                                           int(boxes_train[i, 3])),
                                          (0, 0, 255), 2, 1)
                        n += 1
                    n = 0
                    '''
                    for i in ind_neg:
                        if n%15==0:
                            cv2.rectangle(frame_data_cv, (int(boxes_train[i, 0]), int(boxes_train[i, 1])),
                                          (int(boxes_train[i, 2]), int(boxes_train[i, 3])), (0, 255,255), 2, 1)
                        n+=1
                    '''
                show_particles_init = False
                if show_particles_init:
                    for i in range(filter.num_particles):
                        cx = filter.particles[i, 0]
                        cy = filter.particles[i, 1]
                        cv2.circle(frame_data_cv, (int(cx), int(cy)),
                                   1, (0, 255, 0),
                                   thickness=1)
                show_frame = False
                cv2.circle(frame_data_cv,
                           (int(filter.cur_c[0]), int(filter.cur_c[1])),
                           2, (0, 0, 255),
                           thickness=1)
                if show_frame:
                    cv2.imshow(sequence, frame_data_cv)
                    c = cv2.waitKey(1)
                    # print 'You press: ',chr(c)
                    # if chr(c)=='c':
                    if c != -1:
                        cv2.destroyWindow(sequence)
                        # conf_hist=np.array(conf_hist)
                        # iou_hist=np.array(iou_hist)
                        # np.save('conf_hist.npy',conf_hist)
                        # np.save('iou_hist.npy',iou_hist)
                        break
            end_time = time.time()
            print "Average FPS: %f" % (nFrame / (end_time - start_time))
            log_file.write("Average FPS: %f\n" % (nFrame /
                                                  (end_time - start_time)))
            conf_hist = np.array(conf_hist)
            iou_hist = np.array(iou_hist)
            area_hist = np.array(area_hist)
            pred_hist = np.vstack(pred_hist)
            precisions, auc_pre = utils.calc_prec(gt_boxes, pred_hist)
            # plt.figure()
            # plt.subplot(221)
            # plt.plot(precisions)
            # plt.gca().invert_xaxis()
            # plt.title("Precision plot")
            # plt.xlabel('Location error threshold')
            # plt.ylabel('Precision')
            # plt.yticks(np.linspace(0,1,11))
            # plt.subplot(222)
            # plt.show()
            suc, auc_iou = utils.calc_success(iou_hist)
            records_precision.append(precisions * nFrame)
            records_success.append(suc * nFrame)
            # plt.plot(suc)
            # plt.gca().invert_xaxis()
            # plt.title('Success plot')
            # plt.xlabel('Overlap threshold')
            # plt.ylabel('Success Rate')
            # plt.yticks(np.linspace(0,1,11))
            # plt.show()

            # np.save('conf_hist.npy', conf_hist)
            # np.save('iou_hist.npy', iou_hist)
            # np.save('area_hist.npy',area_hist)
            # print 'Average iou is: %f'%(np.mean(iou_hist))
            print 'Precision @20 is: %f' % precisions[19]
            print 'Auc of Precision is: %f' % auc_pre
            print 'Auc of Success is: %f' % auc_iou
            print 'Reinit times: %d' % reinit
            log_file.write("Precision @20 is: %f\n" % precisions[19])
            log_file.write('Auc of Precision is: %f\n' % auc_pre)
            log_file.write('Auc of Success is: %f\n' % auc_iou)
            log_file.write('Reinit times: %d\n' % reinit)
    log_file.close()
    pkl = open(
        '/home/ccjiang/Documents/caffe-fast-rcnn/examples/tracker/results_100.pkl',
        'w')
    pickle.dump([records_precision, records_success], pkl)
    pkl.close()