def test_integration_single_timeout(self, mock_davis): dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS') with DavisInteractiveSession( davis_root=dataset_dir, subset='train', max_nb_interactions=None, max_time=1, report_save_dir=tempfile.mkdtemp()) as session: count = 0 while session.next(): seq, scribble, new_seq = session.get_scribbles(only_last=True) assert new_seq == (count == 0) assert seq == 'bear' with dataset_dir.joinpath('Scribbles', 'bear', '001.json').open() as fp: sc = json.load(fp) assert sc == scribble assert not is_empty(scribble) # Simulate model predicting masks pred_masks = np.zeros((2, 480, 854)) time.sleep(1.2) session.submit_masks(pred_masks) count += 1 assert count == 1 assert mock_davis.call_count == 0
def test_load_scribble(self): dataset_dir = Path(__file__).parent / 'test_data' / 'DAVIS' davis = Davis(dataset_dir) scribble = davis.load_scribble('bear', 1) assert scribble['sequence'] == 'bear' assert not is_empty(scribble) assert annotated_frames(scribble) == [39]
def test_integration_single(self, mock_davis): dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS') tmp_dir = Path(tempfile.mkdtemp()) with DavisInteractiveSession( davis_root=dataset_dir, subset='train', max_nb_interactions=5, report_save_dir=tmp_dir, max_time=None) as session: count = 0 temp_csv = tmp_dir / ("%s.tmp.csv" % session.report_name) final_csv = tmp_dir / ("%s.csv" % session.report_name) while session.next(): assert not final_csv.exists() assert temp_csv.exists() df = pd.read_csv(temp_csv, index_col=0) assert df.shape == (count * 2, 10) seq, scribble, new_seq = session.get_scribbles(only_last=True) assert new_seq == (count == 0) assert seq == 'bear' if count == 0: with dataset_dir.joinpath('Scribbles', 'bear', '001.json').open() as fp: sc = json.load(fp) assert sc == scribble else: assert annotated_frames(scribble) == [1] assert not is_empty(scribble) assert len(scribble['scribbles']) == 2 assert len(scribble['scribbles'][1]) > 0 assert len(scribble['scribbles'][0]) == 0 # Simulate model predicting masks pred_masks = np.zeros((2, 480, 854)) session.submit_masks( pred_masks, next_scribble_frame_candidates=[1]) if count > 0: assert df.sequence.unique() == ['bear'] assert np.all(df.interaction.unique() == [i + 1 for i in range(count)]) assert np.all(df.object_id.unique() == [1]) count += 1 assert count == 5 assert final_csv.exists() assert not temp_csv.exists() assert mock_davis.call_count == 0
def test_starting_scribble(self, _): dataset_dir = Path(__file__).parent.parent.joinpath( 'dataset', 'test_data', 'DAVIS') service = EvaluationService('train', davis_root=dataset_dir) service.get_samples() scribble = service.get_scribble('bear', 1) assert scribble['sequence'] == 'bear' assert not is_empty(scribble) assert annotated_frames(scribble) == [39]
def test_interaction_equal(self): nb_frames, h, w = 10, 300, 500 gt_empty = np.zeros((nb_frames, h, w), dtype=np.int) gt_empty[0, 100:200, 100:200] = 1 pred_empty = gt_empty.copy() robot = InteractiveScribblesRobot() scribble = robot.interact('test', pred_empty, gt_empty) assert is_empty(scribble) assert annotated_frames(scribble) == [] assert len(scribble['scribbles']) == nb_frames
def test_interaction(self): nb_frames, h, w = 10, 300, 500 gt_empty = np.zeros((nb_frames, h, w), dtype=np.int) pred_empty = gt_empty.copy() gt_empty[5, 100:200, 100:200] = 1 robot = InteractiveScribblesRobot() scribble = robot.interact('test', pred_empty, gt_empty) assert not is_empty(scribble) assert annotated_frames(scribble) == [5] assert len(scribble['scribbles']) == nb_frames lines = scribble['scribbles'][5] for l in lines: assert l['object_id'] == 1 path = np.asarray(l['path']) x, y = path[:, 0], path[:, 1] assert np.all((x >= .2) & (x <= .4)) assert np.all((y >= 1 / 3) & (y <= 2 / 3))
def test_integration_single_only_last(self, mock_davis): dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS') with DavisInteractiveSession( davis_root=dataset_dir, subset='train', max_nb_interactions=4, report_save_dir=tempfile.mkdtemp(), max_time=None) as session: count = 0 annotated_frames_list = [] while session.next(): seq, scribble, new_seq = session.get_scribbles(only_last=True) assert new_seq == (count == 0) assert seq == 'blackswan' if count == 0: with dataset_dir.joinpath('Scribbles', 'blackswan', '001.json').open() as fp: sc = json.load(fp) assert sc == scribble else: assert len(annotated_frames(scribble)) == 1 a_fr = annotated_frames(scribble)[0] assert a_fr not in annotated_frames_list annotated_frames_list.append(a_fr) assert not is_empty(scribble) # Simulate model predicting masks pred_masks = np.zeros((6, 480, 854)) session.submit_masks(pred_masks) count += 1 assert count == 4 assert mock_davis.call_count == 0
def test_interaction_false_positive_single_frame(self): nb_frames, h, w = 1, 300, 500 gt_empty = np.zeros((nb_frames, h, w), dtype=np.int) pred_empty = np.ones((nb_frames, h, w), dtype=np.int) gt_empty[0, 100:200, 100:200] = 1 robot = InteractiveScribblesRobot() scribble = robot.interact('test', pred_empty, gt_empty) assert not is_empty(scribble) assert annotated_frames(scribble) == [0] assert len(scribble['scribbles']) == nb_frames lines = scribble['scribbles'][0] assert lines for l in lines: assert l['object_id'] == 0 path = np.asarray(l['path']) x, y = path[:, 0], path[:, 1] inside = (x >= .2) & (x <= .4) & (y >= 1 / 3) & (y <= 2 / 3) assert not np.any(inside)
def test_interactions_limit(self, mock_start_session, mock_get_scribble, mock_get_report, mock_submit_masks): davis_root = '/tmp/DAVIS' with DavisInteractiveSession( davis_root=davis_root, max_nb_interactions=5, report_save_dir=tempfile.mkdtemp(), max_time=None) as session: assert mock_start_session.call_count == 1 for i in range(7): assert session.next() seq, scribbles, new_seq = session.get_scribbles() assert seq == 'bear' assert is_empty(scribbles) if i % 5 == 0: assert new_seq, i session.submit_masks(None) assert mock_get_scribble.call_count == 2 assert mock_submit_masks.call_count == 7
def test_integration_multiple_sequences(self, mock_davis): dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS') with DavisInteractiveSession( davis_root=dataset_dir, subset='train', max_nb_interactions=4, report_save_dir=tempfile.mkdtemp(), max_time=None) as session: count = 0 for seq, scribble, new_seq in session.scribbles_iterator(): assert new_seq == (count == 0 or count == 4 or count == 8) if count < 8: assert seq == 'bear', count else: assert seq == 'tennis', count if count == 0: with dataset_dir.joinpath('Scribbles', 'bear', '001.json').open() as fp: sc = json.load(fp) assert sc == scribble if count == 4: with dataset_dir.joinpath('Scribbles', 'bear', '002.json').open() as fp: sc = json.load(fp) assert sc == scribble if count == 8: with dataset_dir.joinpath('Scribbles', 'tennis', '001.json').open() as fp: sc = json.load(fp) assert sc == scribble assert not is_empty(scribble) # Simulate model predicting masks pred_masks = np.zeros((2, 480, 854)) session.submit_masks(pred_masks) count += 1 assert count == 12 df = session.get_report() assert mock_davis.call_count == 0 assert df.shape == (2 * 4 * 2 * 1 + 4 * 2 * 2, 8) assert np.all(df.jaccard == 0.) global_summary_file = os.path.join(tempfile.mkdtemp(), 'summary.json') summary = session.get_global_summary() self.assertFalse(os.path.exists(global_summary_file)) self.assertTrue('auc' in summary) self.assertTrue('jaccard_at_threshold' in summary) self.assertEqual(summary['jaccard_at_threshold']['threshold'], 60) self.assertTrue('curve' in summary) curve = summary['curve'] self.assertEqual(len(curve['jaccard']), 6) self.assertEqual(len(curve['time']), 6) summary = session.get_global_summary(save_file=global_summary_file) self.assertTrue(os.path.exists(global_summary_file))