def test_integration_single_timeout(self, mock_davis):
        dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS')

        with DavisInteractiveSession(
                davis_root=dataset_dir,
                subset='train',
                max_nb_interactions=None,
                max_time=1,
                report_save_dir=tempfile.mkdtemp()) as session:
            count = 0

            while session.next():
                seq, scribble, new_seq = session.get_scribbles(only_last=True)
                assert new_seq == (count == 0)
                assert seq == 'bear'
                with dataset_dir.joinpath('Scribbles', 'bear',
                                          '001.json').open() as fp:
                    sc = json.load(fp)
                    assert sc == scribble
                assert not is_empty(scribble)

                # Simulate model predicting masks
                pred_masks = np.zeros((2, 480, 854))

                time.sleep(1.2)

                session.submit_masks(pred_masks)

                count += 1

            assert count == 1

        assert mock_davis.call_count == 0
    def test_integration_single(self, mock_davis):
        dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS')

        tmp_dir = Path(tempfile.mkdtemp())

        with DavisInteractiveSession(
                davis_root=dataset_dir,
                subset='train',
                max_nb_interactions=5,
                report_save_dir=tmp_dir,
                max_time=None) as session:
            count = 0

            temp_csv = tmp_dir / ("%s.tmp.csv" % session.report_name)
            final_csv = tmp_dir / ("%s.csv" % session.report_name)

            while session.next():
                assert not final_csv.exists()
                assert temp_csv.exists()

                df = pd.read_csv(temp_csv, index_col=0)
                assert df.shape == (count * 2, 10)

                seq, scribble, new_seq = session.get_scribbles(only_last=True)
                assert new_seq == (count == 0)
                assert seq == 'bear'
                if count == 0:
                    with dataset_dir.joinpath('Scribbles', 'bear',
                                              '001.json').open() as fp:
                        sc = json.load(fp)
                        assert sc == scribble
                else:
                    assert annotated_frames(scribble) == [1]
                    assert not is_empty(scribble)
                    assert len(scribble['scribbles']) == 2
                    assert len(scribble['scribbles'][1]) > 0
                    assert len(scribble['scribbles'][0]) == 0

                # Simulate model predicting masks
                pred_masks = np.zeros((2, 480, 854))

                session.submit_masks(
                    pred_masks, next_scribble_frame_candidates=[1])

                if count > 0:
                    assert df.sequence.unique() == ['bear']
                    assert np.all(df.interaction.unique() ==
                                  [i + 1 for i in range(count)])
                    assert np.all(df.object_id.unique() == [1])

                count += 1
            assert count == 5
            assert final_csv.exists()
            assert not temp_csv.exists()

        assert mock_davis.call_count == 0
    def test_integration_single_only_last(self, mock_davis):
        dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS')

        with DavisInteractiveSession(
                davis_root=dataset_dir,
                subset='train',
                max_nb_interactions=4,
                report_save_dir=tempfile.mkdtemp(),
                max_time=None) as session:
            count = 0

            annotated_frames_list = []

            while session.next():
                seq, scribble, new_seq = session.get_scribbles(only_last=True)
                assert new_seq == (count == 0)
                assert seq == 'blackswan'
                if count == 0:
                    with dataset_dir.joinpath('Scribbles', 'blackswan',
                                              '001.json').open() as fp:
                        sc = json.load(fp)
                        assert sc == scribble
                else:
                    assert len(annotated_frames(scribble)) == 1
                    a_fr = annotated_frames(scribble)[0]
                    assert a_fr not in annotated_frames_list
                    annotated_frames_list.append(a_fr)
                assert not is_empty(scribble)

                # Simulate model predicting masks
                pred_masks = np.zeros((6, 480, 854))

                session.submit_masks(pred_masks)

                count += 1

            assert count == 4

        assert mock_davis.call_count == 0
    def test_integration_multiple_sequences(self, mock_davis):
        dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS')

        with DavisInteractiveSession(
                davis_root=dataset_dir,
                subset='train',
                max_nb_interactions=4,
                report_save_dir=tempfile.mkdtemp(),
                max_time=None) as session:
            count = 0

            for seq, scribble, new_seq in session.scribbles_iterator():
                assert new_seq == (count == 0 or count == 4 or count == 8)
                if count < 8:
                    assert seq == 'bear', count
                else:
                    assert seq == 'tennis', count
                if count == 0:
                    with dataset_dir.joinpath('Scribbles', 'bear',
                                              '001.json').open() as fp:
                        sc = json.load(fp)
                        assert sc == scribble
                if count == 4:
                    with dataset_dir.joinpath('Scribbles', 'bear',
                                              '002.json').open() as fp:
                        sc = json.load(fp)
                        assert sc == scribble
                if count == 8:
                    with dataset_dir.joinpath('Scribbles', 'tennis',
                                              '001.json').open() as fp:
                        sc = json.load(fp)
                        assert sc == scribble
                assert not is_empty(scribble)

                # Simulate model predicting masks
                pred_masks = np.zeros((2, 480, 854))

                session.submit_masks(pred_masks)

                count += 1

            assert count == 12
            df = session.get_report()

        assert mock_davis.call_count == 0

        assert df.shape == (2 * 4 * 2 * 1 + 4 * 2 * 2, 8)
        assert np.all(df.jaccard == 0.)

        global_summary_file = os.path.join(tempfile.mkdtemp(), 'summary.json')
        summary = session.get_global_summary()
        self.assertFalse(os.path.exists(global_summary_file))
        self.assertTrue('auc' in summary)
        self.assertTrue('jaccard_at_threshold' in summary)
        self.assertEqual(summary['jaccard_at_threshold']['threshold'], 60)
        self.assertTrue('curve' in summary)
        curve = summary['curve']
        self.assertEqual(len(curve['jaccard']), 6)
        self.assertEqual(len(curve['time']), 6)

        summary = session.get_global_summary(save_file=global_summary_file)
        self.assertTrue(os.path.exists(global_summary_file))