def test_integration_single_timeout(self, mock_davis):
dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS')
with DavisInteractiveSession(
davis_root=dataset_dir,
subset='train',
max_nb_interactions=None,
max_time=1,
report_save_dir=tempfile.mkdtemp()) as session:
count = 0
while session.next():
seq, scribble, new_seq = session.get_scribbles(only_last=True)
assert new_seq == (count == 0)
assert seq == 'bear'
with dataset_dir.joinpath('Scribbles', 'bear',
'001.json').open() as fp:
sc = json.load(fp)
assert sc == scribble
assert not is_empty(scribble)
# Simulate model predicting masks
pred_masks = np.zeros((2, 480, 854))
time.sleep(1.2)
session.submit_masks(pred_masks)
count += 1
assert count == 1
assert mock_davis.call_count == 0
def test_integration_single(self, mock_davis):
dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS')
tmp_dir = Path(tempfile.mkdtemp())
with DavisInteractiveSession(
davis_root=dataset_dir,
subset='train',
max_nb_interactions=5,
report_save_dir=tmp_dir,
max_time=None) as session:
count = 0
temp_csv = tmp_dir / ("%s.tmp.csv" % session.report_name)
final_csv = tmp_dir / ("%s.csv" % session.report_name)
while session.next():
assert not final_csv.exists()
assert temp_csv.exists()
df = pd.read_csv(temp_csv, index_col=0)
assert df.shape == (count * 2, 10)
seq, scribble, new_seq = session.get_scribbles(only_last=True)
assert new_seq == (count == 0)
assert seq == 'bear'
if count == 0:
with dataset_dir.joinpath('Scribbles', 'bear',
'001.json').open() as fp:
sc = json.load(fp)
assert sc == scribble
else:
assert annotated_frames(scribble) == [1]
assert not is_empty(scribble)
assert len(scribble['scribbles']) == 2
assert len(scribble['scribbles'][1]) > 0
assert len(scribble['scribbles'][0]) == 0
# Simulate model predicting masks
pred_masks = np.zeros((2, 480, 854))
session.submit_masks(
pred_masks, next_scribble_frame_candidates=[1])
if count > 0:
assert df.sequence.unique() == ['bear']
assert np.all(df.interaction.unique() ==
[i + 1 for i in range(count)])
assert np.all(df.object_id.unique() == [1])
count += 1
assert count == 5
assert final_csv.exists()
assert not temp_csv.exists()
assert mock_davis.call_count == 0
def test_integration_single_only_last(self, mock_davis):
dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS')
with DavisInteractiveSession(
davis_root=dataset_dir,
subset='train',
max_nb_interactions=4,
report_save_dir=tempfile.mkdtemp(),
max_time=None) as session:
count = 0
annotated_frames_list = []
while session.next():
seq, scribble, new_seq = session.get_scribbles(only_last=True)
assert new_seq == (count == 0)
assert seq == 'blackswan'
if count == 0:
with dataset_dir.joinpath('Scribbles', 'blackswan',
'001.json').open() as fp:
sc = json.load(fp)
assert sc == scribble
else:
assert len(annotated_frames(scribble)) == 1
a_fr = annotated_frames(scribble)[0]
assert a_fr not in annotated_frames_list
annotated_frames_list.append(a_fr)
assert not is_empty(scribble)
# Simulate model predicting masks
pred_masks = np.zeros((6, 480, 854))
session.submit_masks(pred_masks)
count += 1
assert count == 4
assert mock_davis.call_count == 0
def test_integration_multiple_sequences(self, mock_davis):
dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS')
with DavisInteractiveSession(
davis_root=dataset_dir,
subset='train',
max_nb_interactions=4,
report_save_dir=tempfile.mkdtemp(),
max_time=None) as session:
count = 0
for seq, scribble, new_seq in session.scribbles_iterator():
assert new_seq == (count == 0 or count == 4 or count == 8)
if count < 8:
assert seq == 'bear', count
else:
assert seq == 'tennis', count
if count == 0:
with dataset_dir.joinpath('Scribbles', 'bear',
'001.json').open() as fp:
sc = json.load(fp)
assert sc == scribble
if count == 4:
with dataset_dir.joinpath('Scribbles', 'bear',
'002.json').open() as fp:
sc = json.load(fp)
assert sc == scribble
if count == 8:
with dataset_dir.joinpath('Scribbles', 'tennis',
'001.json').open() as fp:
sc = json.load(fp)
assert sc == scribble
assert not is_empty(scribble)
# Simulate model predicting masks
pred_masks = np.zeros((2, 480, 854))
session.submit_masks(pred_masks)
count += 1
assert count == 12
df = session.get_report()
assert mock_davis.call_count == 0
assert df.shape == (2 * 4 * 2 * 1 + 4 * 2 * 2, 8)
assert np.all(df.jaccard == 0.)
global_summary_file = os.path.join(tempfile.mkdtemp(), 'summary.json')
summary = session.get_global_summary()
self.assertFalse(os.path.exists(global_summary_file))
self.assertTrue('auc' in summary)
self.assertTrue('jaccard_at_threshold' in summary)
self.assertEqual(summary['jaccard_at_threshold']['threshold'], 60)
self.assertTrue('curve' in summary)
curve = summary['curve']
self.assertEqual(len(curve['jaccard']), 6)
self.assertEqual(len(curve['time']), 6)
summary = session.get_global_summary(save_file=global_summary_file)
self.assertTrue(os.path.exists(global_summary_file))