def draw_box_from_npy(video_path, npy_file, save_path):
    if not os.path.exists(save_path):
        os.makedirs(save_path)

    bboxes = np.load(npy_file)

    frames_files = os.listdir(video_path)
    frames_files.sort(key=str.lower)

    for frame_idx, frame_file in enumerate(frames_files):
        frame = cv2.imread(os.path.join(video_path, frame_file))
        curr_bbox = bboxes[frame_idx]
        im_with_bb = draw_box(frame, curr_bbox)

        filename = os.path.join(save_path, str(frame_idx) + '.jpg')
        cv2.imwrite(filename, im_with_bb)
示例#2
0
    def reset(self, net, domain_specific_nets, train_videos, opts, args):
        self.action_list = []  # a_t,l  # argmax of self.action_prob_list
        self.action_prob_list = []  # output of network (fc6_out)
        self.log_probs_list = [
        ]  # log probs from each self.action_prob_list member
        self.reward_list = []  # tracking score
        self.patch_list = []  # input of network
        self.action_dynamic_list = [
        ]  # action_dynamic used for inference (means before updating the action_dynamic)
        self.result_box_list = []
        self.vid_idx_list = []

        print('generating reinforcement learning dataset')
        transform = ADNet_Augmentation(opts)

        self.env = TrackingEnvironment(train_videos,
                                       opts,
                                       transform=transform,
                                       args=args)
        clip_idx = 0
        while True:  # for every clip (l)

            num_step_history = []  # T_l

            num_frame = 1  # the first frame won't be tracked..
            t = 0
            box_history_clip = []  # for checking oscillation in a clip

            if args.cuda:
                net.module.reset_action_dynamic()
            else:
                net.reset_action_dynamic(
                )  # action dynamic should be in a clip (what makes sense...)

            while True:  # for every frame in a clip (t)
                tic = time.time()

                if args.display_images:
                    im_with_bb = display_result(self.env.get_current_img(),
                                                self.env.get_state())
                    cv2.imshow('patch',
                               self.env.get_current_patch_unprocessed())
                    cv2.waitKey(1)
                else:
                    im_with_bb = draw_box(self.env.get_current_img(),
                                          self.env.get_state())

                if args.save_result_images:
                    cv2.imwrite(
                        'images/' + str(clip_idx) + '-' + str(t) + '.jpg',
                        im_with_bb)

                curr_patch = self.env.get_current_patch()
                if args.cuda:
                    curr_patch = curr_patch.cuda()

                # self.patch_list.append(curr_patch.cpu().data.numpy())  # TODO: saving patch takes cuda memory

                # TODO: saving action_dynamic takes cuda memory
                # if args.cuda:
                #     self.action_dynamic_list.append(net.module.get_action_dynamic())
                # else:
                #     self.action_dynamic_list.append(net.get_action_dynamic())

                curr_patch = curr_patch.unsqueeze(
                    0)  # 1 batch input [1, curr_patch.shape]

                # load ADNetDomainSpecific with video index
                if args.multidomain:
                    vid_idx = self.env.get_current_train_vid_idx()
                else:
                    vid_idx = 0
                if args.cuda:
                    net.module.load_domain_specific(
                        domain_specific_nets[vid_idx])
                else:
                    net.load_domain_specific(domain_specific_nets[vid_idx])

                fc6_out, fc7_out = net.forward(curr_patch,
                                               update_action_dynamic=True)

                if args.cuda:
                    action = np.argmax(fc6_out.detach().cpu().numpy()
                                       )  # TODO: really okay to detach?
                    action_prob = fc6_out.detach().cpu().numpy()[0][action]
                else:
                    action = np.argmax(fc6_out.detach().numpy()
                                       )  # TODO: really okay to detach?
                    action_prob = fc6_out.detach().numpy()[0][action]

                m = Categorical(probs=fc6_out)
                action_ = m.sample(
                )  # action and action_ are same value. Only differ in the type (int and tensor)

                self.log_probs_list.append(
                    m.log_prob(action_).cpu().data.numpy())
                self.vid_idx_list.append(vid_idx)

                self.action_list.append(action)
                # TODO: saving action_prob_list takes cuda memory
                # self.action_prob_list.append(action_prob)

                new_state, reward, done, info = self.env.step(action)

                # check oscilating
                if any((np.array(new_state).round() == x).all()
                       for x in np.array(box_history_clip).round()):
                    action = opts['stop_action']
                    reward, done, finish_epoch = self.env.go_to_next_frame()
                    info['finish_epoch'] = finish_epoch

                # check if number of action is already too much
                if t > opts['num_action_step_max']:
                    action = opts['stop_action']
                    reward, done, finish_epoch = self.env.go_to_next_frame()
                    info['finish_epoch'] = finish_epoch

                # TODO: saving result_box takes cuda memory
                # self.result_box_list.append(list(new_state))
                box_history_clip.append(list(new_state))

                t += 1

                if action == opts['stop_action']:
                    num_frame += 1
                    num_step_history.append(t)
                    t = 0

                toc = time.time() - tic
                print('forward time (clip ' + str(clip_idx) + " - frame " +
                      str(num_frame) + " - t " + str(t) + ") = " + str(toc) +
                      " s")

                if done:  # if finish the clip
                    break

            tracking_scores_size = np.array(num_step_history).sum()
            tracking_scores = np.full(
                tracking_scores_size,
                reward)  # seems no discount factor whatsoever

            self.reward_list.extend(tracking_scores)
            # self.reward_list.append(tracking_scores)

            clip_idx += 1

            if info['finish_epoch']:
                break

        print('generating reinforcement learning dataset finish')
示例#3
0
def adnet_test(net, vid_path, opts, args):

    if torch.cuda.is_available():
        if args.cuda:
            torch.set_default_tensor_type('torch.cuda.FloatTensor')
        if not args.cuda:
            print(
                "WARNING: It looks like you have a CUDA device, but aren't " +
                "using CUDA.\nRun with --cuda for optimal training speed.")
            torch.set_default_tensor_type('torch.FloatTensor')
    else:
        torch.set_default_tensor_type('torch.FloatTensor')

    transform = ADNet_Augmentation(opts)

    print('Testing sequences in ' + str(vid_path) + '...')
    t_sum = 0

    if args.visualize:
        writer = SummaryWriter(
            log_dir=os.path.join('tensorboardx_log', 'online_adapatation_' +
                                 args.save_result_npy))

    ################################
    # Load video sequences
    ################################

    vid_info = {'gt': [], 'img_files': [], 'nframes': 0}

    vid_info['img_files'] = glob.glob(os.path.join(vid_path, 'color', '*.jpg'))
    vid_info['img_files'].sort(key=str.lower)

    gt_path = os.path.join(vid_path, 'groundtruth.txt')

    if not os.path.exists(gt_path):
        bboxes = []
        t = 0
        return bboxes, t_sum

    # parse gt
    gtFile = open(gt_path, 'r')
    gt = gtFile.read().split('\n')
    for i in range(len(gt)):
        if gt[i] == '' or gt[i] is None:
            continue

        if ',' in gt[i]:
            separator = ','
        elif '\t' in gt[i]:
            separator = '\t'
        elif ' ' in gt[i]:
            separator = ' '
        else:
            separator = ','

        gt[i] = gt[i].split(separator)
        gt[i] = list(map(float, gt[i]))
    gtFile.close()

    if len(gt[0]) >= 6:
        for gtidx in range(len(gt)):
            if gt[gtidx] == "":
                continue
            x = gt[gtidx][0:len(gt[gtidx]):2]
            y = gt[gtidx][1:len(gt[gtidx]):2]
            gt[gtidx] = [min(x), min(y), max(x) - min(x), max(y) - min(y)]

    vid_info['gt'] = gt
    if vid_info['gt'][-1] == '':  # small hack
        vid_info['gt'] = vid_info['gt'][:-1]
    vid_info['nframes'] = min(len(vid_info['img_files']), len(vid_info['gt']))

    # catch the first box
    curr_bbox = vid_info['gt'][0]

    # init containers
    bboxes = np.zeros(np.array(
        vid_info['gt']).shape)  # tracking result containers

    ntraining = 0

    # setup training
    if args.cuda:
        optimizer = optim.SGD([{
            'params': net.module.base_network.parameters(),
            'lr': 0
        }, {
            'params': net.module.fc4_5.parameters()
        }, {
            'params': net.module.fc6.parameters()
        }, {
            'params': net.module.fc7.parameters(),
            'lr': 1e-3
        }],
                              lr=1e-3,
                              momentum=opts['train']['momentum'],
                              weight_decay=opts['train']['weightDecay'])
    else:
        optimizer = optim.SGD([{
            'params': net.base_network.parameters(),
            'lr': 0
        }, {
            'params': net.fc4_5.parameters()
        }, {
            'params': net.fc6.parameters()
        }, {
            'params': net.fc7.parameters(),
            'lr': 1e-3
        }],
                              lr=1e-3,
                              momentum=opts['train']['momentum'],
                              weight_decay=opts['train']['weightDecay'])

    action_criterion = nn.CrossEntropyLoss()
    score_criterion = nn.CrossEntropyLoss()

    dataset_storage_pos = None
    dataset_storage_neg = None
    is_negative = False  # is_negative = True if the tracking failed
    target_score = 0
    all_iteration = 0
    t = 0

    for idx in range(vid_info['nframes']):
        # for frame_idx, frame_path in enumerate(vid_info['img_files']):
        frame_idx = idx
        frame_path = vid_info['img_files'][idx]
        t0_wholetracking = time.time()
        frame = cv2.imread(frame_path)

        # draw box or with display, then save
        if args.display_images:
            im_with_bb = display_result(frame,
                                        curr_bbox)  # draw box and display
        else:
            im_with_bb = draw_box(frame, curr_bbox)

        if args.save_result_images:
            filename = os.path.join(args.save_result_images,
                                    str(frame_idx) + '-' + str(t) + '.jpg')
            cv2.imwrite(filename, im_with_bb)

        curr_bbox_old = curr_bbox
        cont_negatives = 0

        if frame_idx > 0:
            # tracking
            if args.cuda:
                net.module.set_phase('test')
            else:
                net.set_phase('test')
            t = 0
            while True:
                curr_patch, curr_bbox, _, _ = transform(
                    frame, curr_bbox, None, None)
                if args.cuda:
                    curr_patch = curr_patch.cuda()

                curr_patch = curr_patch.unsqueeze(
                    0)  # 1 batch input [1, curr_patch.shape]

                fc6_out, fc7_out = net.forward(curr_patch)

                curr_score = fc7_out.detach().cpu().numpy()[0][1]

                if ntraining > args.believe_score_result:
                    if curr_score < opts['failedThre']:
                        cont_negatives += 1

                if args.cuda:
                    action = np.argmax(fc6_out.detach().cpu().numpy()
                                       )  # TODO: really okay to detach?
                    action_prob = fc6_out.detach().cpu().numpy()[0][action]
                else:
                    action = np.argmax(fc6_out.detach().numpy()
                                       )  # TODO: really okay to detach?
                    action_prob = fc6_out.detach().numpy()[0][action]

                # do action
                curr_bbox = do_action(curr_bbox, opts, action, frame.shape)

                # bound the curr_bbox size
                if curr_bbox[2] < 10:
                    curr_bbox[0] = min(
                        0, curr_bbox[0] + curr_bbox[2] / 2 - 10 / 2)
                    curr_bbox[2] = 10
                if curr_bbox[3] < 10:
                    curr_bbox[1] = min(
                        0, curr_bbox[1] + curr_bbox[3] / 2 - 10 / 2)
                    curr_bbox[3] = 10

                t += 1

                # draw box or with display, then save
                if args.display_images:
                    im_with_bb = display_result(
                        frame, curr_bbox)  # draw box and display
                else:
                    im_with_bb = draw_box(frame, curr_bbox)

                if args.save_result_images:
                    filename = os.path.join(
                        args.save_result_images,
                        str(frame_idx) + '-' + str(t) + '.jpg')
                    cv2.imwrite(filename, im_with_bb)

                if action == opts[
                        'stop_action'] or t >= opts['num_action_step_max']:
                    break

            print('final curr_score: %.4f' % curr_score)

            # redetection when confidence < threshold 0.5. But when fc7 is already reliable. Else, just trust the ADNet
            if ntraining > args.believe_score_result:
                if curr_score < 0.5:
                    print('redetection')
                    is_negative = True

                    # redetection process
                    redet_samples = gen_samples(
                        'gaussian', curr_bbox_old, opts['redet_samples'], opts,
                        min(1.5, 0.6 * 1.15**cont_negatives),
                        opts['redet_scale_factor'])
                    score_samples = []

                    for redet_sample in redet_samples:
                        temp_patch, temp_bbox, _, _ = transform(
                            frame, redet_sample, None, None)
                        if args.cuda:
                            temp_patch = temp_patch.cuda()

                        temp_patch = temp_patch.unsqueeze(
                            0)  # 1 batch input [1, curr_patch.shape]

                        fc6_out_temp, fc7_out_temp = net.forward(temp_patch)

                        score_samples.append(
                            fc7_out_temp.detach().cpu().numpy()[0][1])

                    score_samples = np.array(score_samples)
                    max_score_samples_idx = np.argmax(score_samples)

                    # replace the curr_box with the samples with maximum score
                    curr_bbox = redet_samples[max_score_samples_idx]

                    # update the final result image
                    if args.display_images:
                        im_with_bb = display_result(
                            frame, curr_bbox)  # draw box and display
                    else:
                        im_with_bb = draw_box(frame, curr_bbox)

                    if args.save_result_images:
                        filename = os.path.join(args.save_result_images,
                                                str(frame_idx) + '-redet.jpg')
                        cv2.imwrite(filename, im_with_bb)
                else:
                    is_negative = False
            else:
                is_negative = False

        if args.save_result_images:
            filename = os.path.join(args.save_result_images,
                                    'final-' + str(frame_idx) + '.jpg')
            cv2.imwrite(filename, im_with_bb)

        # record the curr_bbox result
        bboxes[frame_idx] = curr_bbox

        # create or update storage + set iteration_range for training
        if frame_idx == 0:
            dataset_storage_pos = OnlineAdaptationDatasetStorage(
                initial_frame=frame,
                first_box=curr_bbox,
                opts=opts,
                args=args,
                positive=True)
            if opts['nNeg_init'] != 0:  # (thanks to small hack in adnet_test) the nNeg_online is also 0
                dataset_storage_neg = OnlineAdaptationDatasetStorage(
                    initial_frame=frame,
                    first_box=curr_bbox,
                    opts=opts,
                    args=args,
                    positive=False)

            iteration_range = range(opts['finetune_iters'])
        else:
            assert dataset_storage_pos is not None
            if opts['nNeg_init'] != 0:  # (thanks to small hack in adnet_test) the nNeg_online is also 0
                assert dataset_storage_neg is not None

            # if confident or when always generate samples, generate new samples
            if ntraining < args.believe_score_result:
                always_generate_samples = True  # as FC7 wasn't trained, it is better to wait for some time to believe its confidence result to decide whether to generate samples or not.. Before believe it, better to just generate sample always
            else:
                always_generate_samples = False

            if always_generate_samples or (not is_negative or
                                           target_score > opts['successThre']):
                dataset_storage_pos.add_frame_then_generate_samples(
                    frame, curr_bbox)

            iteration_range = range(opts['finetune_iters_online'])

        # training when depend on the frequency.. else, don't run the training code...
        if False and frame_idx % args.online_adaptation_every_I_frames == 0:
            ntraining += 1
            # generate dataset just before training
            dataset_pos = OnlineAdaptationDataset(dataset_storage_pos)
            data_loader_pos = data.DataLoader(dataset_pos,
                                              opts['minibatch_size'],
                                              num_workers=args.num_workers,
                                              shuffle=True,
                                              pin_memory=False)
            batch_iterator_pos = None

            if opts['nNeg_init'] != 0:  # (thanks to small hack in adnet_test) the nNeg_online is also 0
                dataset_neg = OnlineAdaptationDataset(dataset_storage_neg)
                data_loader_neg = data.DataLoader(dataset_neg,
                                                  opts['minibatch_size'],
                                                  num_workers=args.num_workers,
                                                  shuffle=True,
                                                  pin_memory=False)
                batch_iterator_neg = None
            else:
                dataset_neg = []

            epoch_size_pos = len(dataset_pos) // opts['minibatch_size']
            epoch_size_neg = len(dataset_neg) // opts['minibatch_size']
            epoch_size = epoch_size_pos + epoch_size_neg  # 1 epoch, how many iterations

            which_dataset = list(np.full(epoch_size_pos, fill_value=1))
            which_dataset.extend(np.zeros(epoch_size_neg, dtype=int))
            shuffle(which_dataset)

            print("1 epoch = " + str(epoch_size) + " iterations")

            if args.cuda:
                net.module.set_phase('train')
            else:
                net.set_phase('train')

            # training loop
            for iteration in iteration_range:
                all_iteration += 1  # use this for update the visualization
                # create batch iterator
                if (not batch_iterator_pos) or (iteration % epoch_size == 0):
                    batch_iterator_pos = iter(data_loader_pos)

                if opts['nNeg_init'] != 0:
                    if (not batch_iterator_neg) or (iteration % epoch_size
                                                    == 0):
                        batch_iterator_neg = iter(data_loader_neg)

                # load train data
                if which_dataset[iteration %
                                 len(which_dataset)]:  # if positive
                    images, bbox, action_label, score_label = next(
                        batch_iterator_pos)
                else:
                    images, bbox, action_label, score_label = next(
                        batch_iterator_neg)

                if args.cuda:
                    images = torch.Tensor(images.cuda())
                    bbox = torch.Tensor(bbox.cuda())
                    action_label = torch.Tensor(action_label.cuda())
                    score_label = torch.Tensor(score_label.float().cuda())

                else:
                    images = torch.Tensor(images)
                    bbox = torch.Tensor(bbox)
                    action_label = torch.Tensor(action_label)
                    score_label = torch.Tensor(score_label)

                # forward
                t0 = time.time()
                action_out, score_out = net(images)

                # backprop
                optimizer.zero_grad()
                if which_dataset[iteration %
                                 len(which_dataset)]:  # if positive
                    action_l = action_criterion(action_out,
                                                torch.max(action_label, 1)[1])
                else:
                    action_l = torch.Tensor([0])
                score_l = score_criterion(score_out, score_label.long())
                loss = action_l + score_l
                loss.backward()
                optimizer.step()
                t1 = time.time()

                if all_iteration % 10 == 0:
                    print('Timer: %.4f sec.' % (t1 - t0))
                    print('iter ' + repr(all_iteration) + ' || Loss: %.4f ||' %
                          (loss.data.item()),
                          end=' ')
                    if args.visualize and args.send_images_to_visualization:
                        random_batch_index = np.random.randint(images.size(0))
                        writer.add_image(
                            'image',
                            images.data[random_batch_index].cpu().numpy(),
                            random_batch_index)

                if args.visualize:
                    writer.add_scalars(
                        'data/iter_loss', {
                            'action_loss': action_l.item(),
                            'score_loss': score_l.item(),
                            'total': (action_l.item() + score_l.item())
                        },
                        global_step=all_iteration)

        t1_wholetracking = time.time()
        t_sum += t1_wholetracking - t0_wholetracking
        print('whole tracking time = %.4f sec.' %
              (t1_wholetracking - t0_wholetracking))

    # evaluate the precision
    bboxes = np.array(bboxes)
    vid_info['gt'] = np.array(vid_info['gt'])

    # iou_precisions = iou_precision_plot(bboxes, vid_info['gt'], vid_path, show=args.display_images, save_plot=args.save_result_images)
    #
    # distance_precisions = distance_precision_plot(bboxes, vid_info['gt'], vid_path, show=args.display_images, save_plot=args.save_result_images)
    #
    # precisions = [distance_precisions, iou_precisions]

    np.save(args.save_result_npy + '-bboxes.npy', bboxes)
    np.save(args.save_result_npy + '-ground_truth.npy', vid_info['gt'])

    # return bboxes, t_sum, precisions
    return bboxes, t_sum