def test_raises_error_if_duplicate_sequence(self):
self.make_submission({'000000': [], '000001': []}, 'folder_a')
self.make_submission({'000000': [], '000002': []}, 'folder_b')
sequence_names = {'000000', '000001', '000002'}
with self.assertRaises(ValueError) as cm:
submission_loader.read_submission(self.temp_dir, sequence_names)
msg = str(cm.exception)
self.assertIn('folder_a/000000.json', msg)
self.assertIn('folder_b/000000.json', msg)
def test_generators_slice_sequences(self):
def make_det(idx):
return {
'label_probs': [0.1, 0.2, 0.3, 0.4],
'bbox': [12, 14, 55 + idx, 46 + idx],
'covars': [[[0, 0], [0, 0]], [[0, 0], [0, 0]]]
}
submission = {
'000000': [
[make_det(idx) for idx in range(3)],
[make_det(idx) for idx in range(5)],
[make_det(idx) for idx in range(7)],
[make_det(idx) for idx in range(9)]
],
'000001': [
[make_det(idx) for idx in range(2)],
[make_det(idx) for idx in range(4)],
[make_det(idx) for idx in range(6)]
]
}
self.make_submission(submission)
sequences = submission_loader.read_submission(self.temp_dir, {'000000', '000001'}, start_index=1, end_index=5)
for sequence_name in submission.keys():
self.assertIn(sequence_name, sequences)
seq_dets = list(sequences[sequence_name])
detections = submission[sequence_name][1:5]
self.assertEqual(len(detections), len(seq_dets))
for img_idx in range(len(seq_dets)):
img_dets = list(seq_dets[img_idx])
self.assertEqual(len(detections[img_idx]), len(img_dets))
for det_idx in range(len(img_dets)):
self.assertIsInstance(img_dets[det_idx], data_holders.DetectionInstance)
self.assertNPEqual(img_dets[det_idx].box, detections[img_idx][det_idx]['bbox'])
def test_returns_map_of_name_to_generator(self):
self.make_submission({'000000': [], '000001': []})
sequences = submission_loader.read_submission(self.temp_dir, {'000000', '000001'})
self.assertIn('000000', sequences)
self.assertIn('000001', sequences)
self.assertIsInstance(sequences['000000'], types.GeneratorType)
self.assertIsInstance(sequences['000001'], types.GeneratorType)
def do_evaluation(submission_dir, ground_truth_dir):
"""
Evaluate a particular image sequence
:param submission_dir:
:param ground_truth_dir:
:return:
"""
ground_truth = gt_loader.read_ground_truth(ground_truth_dir)
detections = submission_loader.read_submission(submission_dir,
expected_sequence_names=set(
ground_truth.keys()))
matches = gt_loader.match_sequences(ground_truth, detections)
evaluator = PDQ()
score = evaluator.score(matches)
TP, FP, FN = evaluator.get_assignment_counts()
avg_spatial_quality = evaluator.get_avg_spatial_score()
avg_label_quality = evaluator.get_avg_label_score()
avg_overall_quality = evaluator.get_avg_overall_quality_score()
return {
'score': score * 100,
'avg_spatial': avg_spatial_quality,
'avg_label': avg_label_quality,
'avg_pPDQ': avg_overall_quality,
'TPs': TP,
'FPs': FP,
'FNs': FN
}
def test_generators_read_sequences(self):
submission = {
'000000': [
[{
'label_probs': [0.1, 0.2, 0.3, 0.4],
'bbox': [12, 14, 55, 46],
'covars': [[[0, 0], [0, 0]], [[0, 0], [0, 0]]]
}],
[],
[],
[{
'label_probs': [0.1, 0.2, 0.3, 0.4],
'bbox': [12, 14, 55, 46],
'covars': [[[0, 0], [0, 0]], [[0, 0], [0, 0]]]
}, {
'label_probs': [0.1, 0.2, 0.3, 0.4],
'bbox': [12, 14, 55, 46],
'covars': [[[0, 0], [0, 0]], [[0, 0], [0, 0]]]
}],
],
'000001': [
[{
'label_probs': [0.1, 0.2, 0.3, 0.4],
'bbox': [12, 14, 55, 46],
'covars': [[[0, 0], [0, 0]], [[0, 0], [0, 0]]]
}],
[],
[],
[{
'label_probs': [0.1, 0.2, 0.3, 0.4],
'bbox': [12, 14, 55, 46],
'covars': [[[0, 0], [0, 0]], [[0, 0], [0, 0]]]
}, {
'label_probs': [0.1, 0.2, 0.3, 0.4],
'bbox': [12, 14, 55, 46],
'covars': [[[0, 0], [0, 0]], [[0, 0], [0, 0]]]
}],
]
}
self.make_submission(submission)
sequences = submission_loader.read_submission(self.temp_dir,
{'000000', '000001'})
for sequence_name, detections in submission.items():
self.assertIn(sequence_name, sequences)
seq_dets = list(sequences[sequence_name])
self.assertEqual(len(detections), len(seq_dets))
for img_idx in range(len(seq_dets)):
img_dets = list(seq_dets[img_idx])
self.assertEqual(len(detections[img_idx]), len(img_dets))
for det_idx in range(len(img_dets)):
self.assertIsInstance(img_dets[det_idx],
data_holders.DetectionInstance)
expected_probs = detections[img_idx][det_idx][
'label_probs']
expected_list = np.zeros(len(class_list.CLASSES),
dtype=np.float32)
expected_list[0:len(expected_probs)] = expected_probs
self.assertNPEqual(expected_list,
img_dets[det_idx].class_list)
def test_searches_for_sequences_in_subfolders(self):
self.make_submission({'000000': [], '000001': []}, 'folder_a')
self.make_submission({'000002': [], '000003': []}, 'folder_b')
sequence_names = {'000000', '000001', '000002', '000003'}
sequences = submission_loader.read_submission(self.temp_dir, sequence_names)
self.assertEqual(sequence_names, set(sequences.keys()))
self.assertIsInstance(sequences['000000'], types.GeneratorType)
self.assertIsInstance(sequences['000001'], types.GeneratorType)
self.assertIsInstance(sequences['000002'], types.GeneratorType)
self.assertIsInstance(sequences['000003'], types.GeneratorType)
def do_evaluation(submission_dir,
ground_truth_dir,
sequences=None,
num_frames=-1,
start_frame=0):
"""
Evaluate a particular image sequence
:param submission_dir: location of the detections .json files (one for each sequence)
:param ground_truth_dir: location of the ground-truth folders (one for each sequence).
Each ground-truth folder must contain mask images (.png format) and a matching labels.json file.
:param sequences: A whitelist of sequence ids to include, as integers
:param num_frames: The number of frames to read from each sequence, default is all available.
:param start_frame: The index of the first frame to read
:return: Dictionary containing summary of all metrics used in competition leaderboard
(score, average spatial quality, average label quality, average overall quality (avg_pPDQ),
true positives, false positives, and false negatives)
"""
ground_truth = gt_loader.read_ground_truth(ground_truth_dir,
sequences,
start_index=start_frame,
end_index=start_frame +
num_frames)
detections = submission_loader.read_submission(
submission_dir,
expected_sequence_names=set(ground_truth.keys()),
start_index=start_frame,
end_index=start_frame + num_frames)
matches = gt_loader.match_sequences(ground_truth, detections)
evaluator = PDQ()
score = evaluator.score(matches)
TP, FP, FN = evaluator.get_assignment_counts()
avg_spatial_quality = evaluator.get_avg_spatial_score()
avg_label_quality = evaluator.get_avg_label_score()
avg_overall_quality = evaluator.get_avg_overall_quality_score()
avg_fp_quality = evaluator.get_avg_fp_score()
return {
'score': score * 100,
'avg_spatial': avg_spatial_quality,
'avg_label': avg_label_quality,
'avg_pPDQ': avg_overall_quality,
'avg_fp_quality': avg_fp_quality,
'TPs': TP,
'FPs': FP,
'FNs': FN
}
def test_finds_only_requested_sequences(self):
self.make_submission({'000000': [], '000001': []})
sequences = submission_loader.read_submission(self.temp_dir, {'000000'})
self.assertEqual({'000000'}, set(sequences.keys()))
self.assertIsInstance(sequences['000000'], types.GeneratorType)
