def test_integration_single_timeout(self, mock_davis):
dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS')
with DavisInteractiveSession(
davis_root=dataset_dir,
subset='train',
max_nb_interactions=None,
max_time=1,
report_save_dir=tempfile.mkdtemp()) as session:
count = 0
while session.next():
seq, scribble, new_seq = session.get_scribbles(only_last=True)
assert new_seq == (count == 0)
assert seq == 'bear'
with dataset_dir.joinpath('Scribbles', 'bear',
'001.json').open() as fp:
sc = json.load(fp)
assert sc == scribble
assert not is_empty(scribble)
# Simulate model predicting masks
pred_masks = np.zeros((2, 480, 854))
time.sleep(1.2)
session.submit_masks(pred_masks)
count += 1
assert count == 1
assert mock_davis.call_count == 0
def test_load_scribble(self):
dataset_dir = Path(__file__).parent / 'test_data' / 'DAVIS'
davis = Davis(dataset_dir)
scribble = davis.load_scribble('bear', 1)
assert scribble['sequence'] == 'bear'
assert not is_empty(scribble)
assert annotated_frames(scribble) == [39]
def test_integration_single(self, mock_davis):
dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS')
tmp_dir = Path(tempfile.mkdtemp())
with DavisInteractiveSession(
davis_root=dataset_dir,
subset='train',
max_nb_interactions=5,
report_save_dir=tmp_dir,
max_time=None) as session:
count = 0
temp_csv = tmp_dir / ("%s.tmp.csv" % session.report_name)
final_csv = tmp_dir / ("%s.csv" % session.report_name)
while session.next():
assert not final_csv.exists()
assert temp_csv.exists()
df = pd.read_csv(temp_csv, index_col=0)
assert df.shape == (count * 2, 10)
seq, scribble, new_seq = session.get_scribbles(only_last=True)
assert new_seq == (count == 0)
assert seq == 'bear'
if count == 0:
with dataset_dir.joinpath('Scribbles', 'bear',
'001.json').open() as fp:
sc = json.load(fp)
assert sc == scribble
else:
assert annotated_frames(scribble) == [1]
assert not is_empty(scribble)
assert len(scribble['scribbles']) == 2
assert len(scribble['scribbles'][1]) > 0
assert len(scribble['scribbles'][0]) == 0
# Simulate model predicting masks
pred_masks = np.zeros((2, 480, 854))
session.submit_masks(
pred_masks, next_scribble_frame_candidates=[1])
if count > 0:
assert df.sequence.unique() == ['bear']
assert np.all(df.interaction.unique() ==
[i + 1 for i in range(count)])
assert np.all(df.object_id.unique() == [1])
count += 1
assert count == 5
assert final_csv.exists()
assert not temp_csv.exists()
assert mock_davis.call_count == 0
def test_starting_scribble(self, _):
dataset_dir = Path(__file__).parent.parent.joinpath(
'dataset', 'test_data', 'DAVIS')
service = EvaluationService('train', davis_root=dataset_dir)
service.get_samples()
scribble = service.get_scribble('bear', 1)
assert scribble['sequence'] == 'bear'
assert not is_empty(scribble)
assert annotated_frames(scribble) == [39]
def test_interaction_equal(self):
nb_frames, h, w = 10, 300, 500
gt_empty = np.zeros((nb_frames, h, w), dtype=np.int)
gt_empty[0, 100:200, 100:200] = 1
pred_empty = gt_empty.copy()
robot = InteractiveScribblesRobot()
scribble = robot.interact('test', pred_empty, gt_empty)
assert is_empty(scribble)
assert annotated_frames(scribble) == []
assert len(scribble['scribbles']) == nb_frames
def test_interaction(self):
nb_frames, h, w = 10, 300, 500
gt_empty = np.zeros((nb_frames, h, w), dtype=np.int)
pred_empty = gt_empty.copy()
gt_empty[5, 100:200, 100:200] = 1
robot = InteractiveScribblesRobot()
scribble = robot.interact('test', pred_empty, gt_empty)
assert not is_empty(scribble)
assert annotated_frames(scribble) == [5]
assert len(scribble['scribbles']) == nb_frames
lines = scribble['scribbles'][5]
for l in lines:
assert l['object_id'] == 1
path = np.asarray(l['path'])
x, y = path[:, 0], path[:, 1]
assert np.all((x >= .2) & (x <= .4))
assert np.all((y >= 1 / 3) & (y <= 2 / 3))
def test_integration_single_only_last(self, mock_davis):
dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS')
with DavisInteractiveSession(
davis_root=dataset_dir,
subset='train',
max_nb_interactions=4,
report_save_dir=tempfile.mkdtemp(),
max_time=None) as session:
count = 0
annotated_frames_list = []
while session.next():
seq, scribble, new_seq = session.get_scribbles(only_last=True)
assert new_seq == (count == 0)
assert seq == 'blackswan'
if count == 0:
with dataset_dir.joinpath('Scribbles', 'blackswan',
'001.json').open() as fp:
sc = json.load(fp)
assert sc == scribble
else:
assert len(annotated_frames(scribble)) == 1
a_fr = annotated_frames(scribble)[0]
assert a_fr not in annotated_frames_list
annotated_frames_list.append(a_fr)
assert not is_empty(scribble)
# Simulate model predicting masks
pred_masks = np.zeros((6, 480, 854))
session.submit_masks(pred_masks)
count += 1
assert count == 4
assert mock_davis.call_count == 0
def test_interaction_false_positive_single_frame(self):
nb_frames, h, w = 1, 300, 500
gt_empty = np.zeros((nb_frames, h, w), dtype=np.int)
pred_empty = np.ones((nb_frames, h, w), dtype=np.int)
gt_empty[0, 100:200, 100:200] = 1
robot = InteractiveScribblesRobot()
scribble = robot.interact('test', pred_empty, gt_empty)
assert not is_empty(scribble)
assert annotated_frames(scribble) == [0]
assert len(scribble['scribbles']) == nb_frames
lines = scribble['scribbles'][0]
assert lines
for l in lines:
assert l['object_id'] == 0
path = np.asarray(l['path'])
x, y = path[:, 0], path[:, 1]
inside = (x >= .2) & (x <= .4) & (y >= 1 / 3) & (y <= 2 / 3)
assert not np.any(inside)
def test_interactions_limit(self, mock_start_session, mock_get_scribble,
mock_get_report, mock_submit_masks):
davis_root = '/tmp/DAVIS'
with DavisInteractiveSession(
davis_root=davis_root,
max_nb_interactions=5,
report_save_dir=tempfile.mkdtemp(),
max_time=None) as session:
assert mock_start_session.call_count == 1
for i in range(7):
assert session.next()
seq, scribbles, new_seq = session.get_scribbles()
assert seq == 'bear'
assert is_empty(scribbles)
if i % 5 == 0:
assert new_seq, i
session.submit_masks(None)
assert mock_get_scribble.call_count == 2
assert mock_submit_masks.call_count == 7
def test_integration_multiple_sequences(self, mock_davis):
dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS')
with DavisInteractiveSession(
davis_root=dataset_dir,
subset='train',
max_nb_interactions=4,
report_save_dir=tempfile.mkdtemp(),
max_time=None) as session:
count = 0
for seq, scribble, new_seq in session.scribbles_iterator():
assert new_seq == (count == 0 or count == 4 or count == 8)
if count < 8:
assert seq == 'bear', count
else:
assert seq == 'tennis', count
if count == 0:
with dataset_dir.joinpath('Scribbles', 'bear',
'001.json').open() as fp:
sc = json.load(fp)
assert sc == scribble
if count == 4:
with dataset_dir.joinpath('Scribbles', 'bear',
'002.json').open() as fp:
sc = json.load(fp)
assert sc == scribble
if count == 8:
with dataset_dir.joinpath('Scribbles', 'tennis',
'001.json').open() as fp:
sc = json.load(fp)
assert sc == scribble
assert not is_empty(scribble)
# Simulate model predicting masks
pred_masks = np.zeros((2, 480, 854))
session.submit_masks(pred_masks)
count += 1
assert count == 12
df = session.get_report()
assert mock_davis.call_count == 0
assert df.shape == (2 * 4 * 2 * 1 + 4 * 2 * 2, 8)
assert np.all(df.jaccard == 0.)
global_summary_file = os.path.join(tempfile.mkdtemp(), 'summary.json')
summary = session.get_global_summary()
self.assertFalse(os.path.exists(global_summary_file))
self.assertTrue('auc' in summary)
self.assertTrue('jaccard_at_threshold' in summary)
self.assertEqual(summary['jaccard_at_threshold']['threshold'], 60)
self.assertTrue('curve' in summary)
curve = summary['curve']
self.assertEqual(len(curve['jaccard']), 6)
self.assertEqual(len(curve['time']), 6)
summary = session.get_global_summary(save_file=global_summary_file)
self.assertTrue(os.path.exists(global_summary_file))