def test_integration_single_timeout(self, mock_davis):
dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS')
with DavisInteractiveSession(
davis_root=dataset_dir,
subset='train',
max_nb_interactions=None,
max_time=1,
report_save_dir=tempfile.mkdtemp()) as session:
count = 0
while session.next():
seq, scribble, new_seq = session.get_scribbles(only_last=True)
assert new_seq == (count == 0)
assert seq == 'bear'
with dataset_dir.joinpath('Scribbles', 'bear',
'001.json').open() as fp:
sc = json.load(fp)
assert sc == scribble
assert not is_empty(scribble)
# Simulate model predicting masks
pred_masks = np.zeros((2, 480, 854))
time.sleep(1.2)
session.submit_masks(pred_masks)
count += 1
assert count == 1
assert mock_davis.call_count == 0
def test_report_folder_creation(self, mock_davis):
dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS')
tmp_dir = Path(tempfile.mkdtemp()) / 'test'
assert not tmp_dir.exists()
session = DavisInteractiveSession(
davis_root=dataset_dir, subset='train', report_save_dir=tmp_dir)
assert tmp_dir.exists()
assert mock_davis.call_count == 0
def test_integration_single(self, mock_davis):
dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS')
tmp_dir = Path(tempfile.mkdtemp())
with DavisInteractiveSession(
davis_root=dataset_dir,
subset='train',
max_nb_interactions=5,
report_save_dir=tmp_dir,
max_time=None) as session:
count = 0
temp_csv = tmp_dir / ("%s.tmp.csv" % session.report_name)
final_csv = tmp_dir / ("%s.csv" % session.report_name)
while session.next():
assert not final_csv.exists()
assert temp_csv.exists()
df = pd.read_csv(temp_csv, index_col=0)
assert df.shape == (count * 2, 10)
seq, scribble, new_seq = session.get_scribbles(only_last=True)
assert new_seq == (count == 0)
assert seq == 'bear'
if count == 0:
with dataset_dir.joinpath('Scribbles', 'bear',
'001.json').open() as fp:
sc = json.load(fp)
assert sc == scribble
else:
assert annotated_frames(scribble) == [1]
assert not is_empty(scribble)
assert len(scribble['scribbles']) == 2
assert len(scribble['scribbles'][1]) > 0
assert len(scribble['scribbles'][0]) == 0
# Simulate model predicting masks
pred_masks = np.zeros((2, 480, 854))
session.submit_masks(
pred_masks, next_scribble_frame_candidates=[1])
if count > 0:
assert df.sequence.unique() == ['bear']
assert np.all(df.interaction.unique() ==
[i + 1 for i in range(count)])
assert np.all(df.object_id.unique() == [1])
count += 1
assert count == 5
assert final_csv.exists()
assert not temp_csv.exists()
assert mock_davis.call_count == 0
def test_load_scribble(self):
dataset_dir = Path(__file__).parent / 'test_data' / 'DAVIS'
davis = Davis(dataset_dir)
scribble = davis.load_scribble('bear', 1)
assert scribble['sequence'] == 'bear'
assert not is_empty(scribble)
assert annotated_frames(scribble) == [39]
def setUpClass(cls):
img_dir = Path(__file__).parent / 'test_files'
masks = []
for i in range(4):
img_path = img_dir / '{:05}.png'.format(i)
masks.append(np.asarray(Image.open(img_path)))
masks = np.stack(masks, axis=0)
assert masks.shape[0] == 4
cls.test_masks = masks
def test_starting_scribble(self, _):
dataset_dir = Path(__file__).parent.parent.joinpath(
'dataset', 'test_data', 'DAVIS')
service = EvaluationService('train', davis_root=dataset_dir)
service.get_samples()
scribble = service.get_scribble('bear', 1)
assert scribble['sequence'] == 'bear'
assert not is_empty(scribble)
assert annotated_frames(scribble) == [39]
def test_shuffle(self, mock_davis):
dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS')
with DavisInteractiveSession(
davis_root=dataset_dir,
subset='train',
shuffle=True,
report_save_dir=tempfile.mkdtemp()) as session:
assert ('bear', 1) in session.samples
assert ('bear', 2) in session.samples
assert ('tennis', 1) in session.samples
assert mock_davis.call_count == 0
def test_integration_single_only_last(self, mock_davis):
dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS')
with DavisInteractiveSession(
davis_root=dataset_dir,
subset='train',
max_nb_interactions=4,
report_save_dir=tempfile.mkdtemp(),
max_time=None) as session:
count = 0
annotated_frames_list = []
while session.next():
seq, scribble, new_seq = session.get_scribbles(only_last=True)
assert new_seq == (count == 0)
assert seq == 'blackswan'
if count == 0:
with dataset_dir.joinpath('Scribbles', 'blackswan',
'001.json').open() as fp:
sc = json.load(fp)
assert sc == scribble
else:
assert len(annotated_frames(scribble)) == 1
a_fr = annotated_frames(scribble)[0]
assert a_fr not in annotated_frames_list
annotated_frames_list.append(a_fr)
assert not is_empty(scribble)
# Simulate model predicting masks
pred_masks = np.zeros((6, 480, 854))
session.submit_masks(pred_masks)
count += 1
assert count == 4
assert mock_davis.call_count == 0
def test_exceptions(self, _):
dataset_dir = Path(__file__).parent.parent.joinpath(
'dataset', 'test_data', 'DAVIS')
service = EvaluationService('train', max_i=1, davis_root=dataset_dir)
service.get_samples()
with self.assertRaises(ValueError):
service.post_predicted_masks('bear', 1, None, 0, 2, None, None)
with self.assertRaises(ValueError):
service.post_predicted_masks('bear', 1, None, 0, 0, None, None)
with self.assertRaises(ValueError):
service.post_predicted_masks('novalidsequence', 1, None, 0, 1,
None, None)
with self.assertRaises(ValueError):
service.post_predicted_masks('bear', 4, None, 0, 1, None, None)
def test_load_annotations(self):
dataset_dir = Path(__file__).parent / 'test_data' / 'DAVIS'
num_frames = Davis.dataset['bear']['num_frames']
Davis.dataset['bear']['num_frames'] = 1
davis = Davis(dataset_dir)
ann = davis.load_annotations('bear')
assert ann.shape == (1, 480, 854)
assert ann.dtype == np.int
assert np.all(np.unique(ann) == np.asarray([0, 1]))
ann2 = davis.load_annotations('bear', dtype=np.uint8)
assert ann2.shape == (1, 480, 854)
assert ann2.dtype == np.uint8
assert np.all(np.unique(ann2) == np.asarray([0, 1]))
assert np.all(ann2.astype(np.int) == ann)
Davis.dataset['bear']['num_frames'] = num_frames
def test_load_images(self):
dataset_dir = Path(__file__).parent / 'test_data' / 'DAVIS'
num_frames = Davis.dataset['bear']['num_frames']
Davis.dataset['bear']['num_frames'] = 1
davis = Davis(dataset_dir)
img = davis.load_images('bear')
assert img.shape == (1, 480, 854, 3)
assert img.dtype == np.uint8
assert img.min() >= 0
assert img.max() <= 255
img2 = davis.load_images('bear', np.int)
assert img2.shape == (1, 480, 854, 3)
assert img2.dtype == np.int
assert img2.min() >= 0
assert img2.max() <= 255
assert np.all(img.astype(np.int) == img2)
Davis.dataset['bear']['num_frames'] = num_frames
def test_integration_multiple_sequences(self, mock_davis):
dataset_dir = Path(__file__).parent.joinpath('test_data', 'DAVIS')
with DavisInteractiveSession(
davis_root=dataset_dir,
subset='train',
max_nb_interactions=4,
report_save_dir=tempfile.mkdtemp(),
max_time=None) as session:
count = 0
for seq, scribble, new_seq in session.scribbles_iterator():
assert new_seq == (count == 0 or count == 4 or count == 8)
if count < 8:
assert seq == 'bear', count
else:
assert seq == 'tennis', count
if count == 0:
with dataset_dir.joinpath('Scribbles', 'bear',
'001.json').open() as fp:
sc = json.load(fp)
assert sc == scribble
if count == 4:
with dataset_dir.joinpath('Scribbles', 'bear',
'002.json').open() as fp:
sc = json.load(fp)
assert sc == scribble
if count == 8:
with dataset_dir.joinpath('Scribbles', 'tennis',
'001.json').open() as fp:
sc = json.load(fp)
assert sc == scribble
assert not is_empty(scribble)
# Simulate model predicting masks
pred_masks = np.zeros((2, 480, 854))
session.submit_masks(pred_masks)
count += 1
assert count == 12
df = session.get_report()
assert mock_davis.call_count == 0
assert df.shape == (2 * 4 * 2 * 1 + 4 * 2 * 2, 8)
assert np.all(df.jaccard == 0.)
global_summary_file = os.path.join(tempfile.mkdtemp(), 'summary.json')
summary = session.get_global_summary()
self.assertFalse(os.path.exists(global_summary_file))
self.assertTrue('auc' in summary)
self.assertTrue('jaccard_at_threshold' in summary)
self.assertEqual(summary['jaccard_at_threshold']['threshold'], 60)
self.assertTrue('curve' in summary)
curve = summary['curve']
self.assertEqual(len(curve['jaccard']), 6)
self.assertEqual(len(curve['time']), 6)
summary = session.get_global_summary(save_file=global_summary_file)
self.assertTrue(os.path.exists(global_summary_file))
def setUpClass(cls):
img_path = Path(__file__).parent / 'test_files' / '00000.png'
img_path = str(img_path.resolve())
mask = np.asarray(Image.open(img_path))
cls.test_mask = mask