def test_load_and_save():
"""
Load, save and load again full and thin analyzer instances, and verify that content is identical.
"""
base_dir = os.path.dirname(__file__)
relative_data_dir = 'data/1_example'
data_dir = os.path.join(base_dir, relative_data_dir)
# ----------------------------
# load and save full analyzer
# ----------------------------
# load analyzer
analyzer_file = os.path.join(data_dir, 'analyzer_full.p')
analyzer_full = Analyzer.load(analyzer_file)
# save analyzer in temporary directory
outupt_dir = os.path.join(data_dir, 'tmp_full')
shutil.rmtree(outupt_dir, ignore_errors=True) # delete existing temp dir
# save instance with image dir
output_file_full = os.path.join(outupt_dir, 'analyzer.p')
analyzer_full.save(output_file_full, save_thin_instance=False, save_images_in_dir=True, image_name_template='{:08d}.png')
# ----------------------------
# load and save thin analyzer
# ----------------------------
analyzer_file = os.path.join(data_dir, 'analyzer_thin.p')
analyzer_thin = Analyzer.load(analyzer_file, load_images_from_dir=True, sfx='png') # use lossless png image, not jpg
# save analyzer in temporary directory
outupt_dir = os.path.join(data_dir, 'tmp_thin')
shutil.rmtree(outupt_dir, ignore_errors=True) # delete existing temp dir
# save instance with image dir
output_file_thin = os.path.join(outupt_dir, 'analyzer.p')
analyzer_thin.save(output_file_thin, save_thin_instance=True, save_images_in_dir=True, image_name_template='{:08d}.png') # use lossless png image, not jpg
# ----------------------------------------------------------------
# load analyzers from saved dirs and compare content thin analyzer
# ----------------------------------------------------------------
# load analyzers
analyzer_full2 = Analyzer.load(output_file_full)
analyzer_thin2 = Analyzer.load(output_file_thin, load_images_from_dir=True, sfx='png') # use lossless png image, not jpg
# compare content
is_equal = compare_instances(analyzer_full2, analyzer_thin2)
assert is_equal
def load_test_analyzer(data_dir='ILSVRC2015_00078000'):
"""
Load analyzer object from saved test data.
Parameters
----------
data_dir : str, optional
Data directory name, should be name of one of the folders inside analyze/tests/data/.
Returns
-------
anaylzer : Analyzer
Analyzer object.
analyzer_root_dir : str
Path of analyzer root directory, such that full images path is the concatenation of analyzer_root_dir and
analyzer.data image_path.
"""
# load reference analyzer
base_dir = os.path.dirname(__file__)
relative_data_dir = os.path.join('data', data_dir)
data_dir = os.path.join(base_dir, relative_data_dir)
analyzer_file = os.path.join(data_dir, 'analyzer.p')
analyzer = Analyzer.load(analyzer_file, load_images_from_dir=False)
analyzer_root_dir = os.path.join(base_dir, '..')
return analyzer, analyzer_root_dir
def test_visualize_example():
base_dir = os.path.dirname(__file__)
relative_data_dir = 'data/ILSVRC2015_00078000'
data_dir = os.path.join(base_dir, relative_data_dir)
analyze_root_dir = os.path.abspath(os.path.join(base_dir, '..'))
os.chdir(analyze_root_dir)
# load analyzer
analyzer_file = os.path.join(data_dir, 'analyzer.p')
analyzer = Analyzer.load(analyzer_file, load_images_from_dir=True)
# visualize example
save_fig_path = os.path.join(base_dir, 'save_fig_example.png')
frame_id = 40
display = False # True
image = analyzer.visualize_example(key=frame_id,
class_names=CLASS_NAMES,
display=display,
save_fig_path=save_fig_path)
# compare to reference image
image_ref_path = os.path.join(
base_dir, 'data/visualizations/visualization_example.png')
image_ref = cv2.imread(image_ref_path, cv2.IMREAD_UNCHANGED)
is_close = np.isclose(image, image_ref,
atol=2) # allow up to 2 gray level difference
is_all_close = np.all(is_close)
assert is_all_close
def test_empty_preds_gts():
base_dir = os.path.dirname(__file__)
relative_data_dir = 'data/ILSVRC2015_00078000'
data_dir = os.path.join(base_dir, relative_data_dir)
# load analyzer
analyzer_file = os.path.join(data_dir, 'analyzer.p')
analyzer = Analyzer.load(analyzer_file, load_images_from_dir=False)
# initialize total confusion matrix
num_classes = len(CLASS_NAMES)
cm_total = np.zeros((num_classes + 1, num_classes + 1))
# iterate over frames
counter = 0
for frame_id, item in analyzer.items():
# unpack data
prediction, ground_truth, image_path, image = analyzer.unpack_item(
item)[0:4]
if counter == 1: # delete predictions
prediction = Box([], image_shape=(100, 200))
elif counter == 2: # delete groud truths
ground_truth = Box([], image_shape=(100, 200))
elif counter == 3: # delete image shape
try:
ground_truth = Box([], image_shape=None)
except ValueError as e:
cond3 = e.__repr__(
) == "ValueError('image_shape must be tuple of length 2 (height, width) or 3 (height, width, channels), got None',)"
# calculate confusion matrix of current frame
cm = ConfusionMatrix.calculate_confusion_matrix(prediction,
ground_truth,
CLASS_NAMES,
normalize=None,
score_th=0.3,
iou_th=0.5,
iou_criterion='all',
score_criterion='all',
display=False)
if counter == 1: # delete predictions
cond1 = cm[:, -1].sum() == 3
elif counter == 2: # delete groud truths
cond2 = cm[-1, :].sum() == 3
# analyze cm
# advance counter
counter += 1
# check conditions
assert cond1
assert cond2
assert cond3
def test_initialize_with_simple_wrapper():
base_dir = os.path.dirname(__file__)
relative_data_dir = 'data/ILSVRC2015_00078000'
data_dir = os.path.join(base_dir, relative_data_dir)
# load analyzer
analyzer_file = os.path.join(data_dir, 'analyzer.p')
analyzer = Analyzer.load(analyzer_file, load_images_from_dir=False)
# initialize total confusion matrix
num_classes = len(CLASS_NAMES)
cm_total = np.zeros((num_classes + 1, num_classes + 1))
def generate_report():
# load reference analyzer
base_dir = os.path.dirname(__file__)
relative_data_dir = '../tests/data/ILSVRC2015_00078000'
data_dir = os.path.join(base_dir, relative_data_dir)
analyzer_file = os.path.join(data_dir, 'analyzer.p')
analyzer = Analyzer.load(analyzer_file, load_images_from_dir=False)
# set output directory
analyzer.output_dir = os.path.join(base_dir, 'output', 'generate_report')
os.makedirs(analyzer.output_dir, exist_ok=True)
# generate report
analyzer.evaluate_performance(generate_report=True)
pass
def test_analyzer_mutable_mapping_implementation():
"""
Analyzer class inherits from collections.abc.MutableMapping, which demands implementation of
several class methods such as __setitem__, __iter__, etc.
Here we will check the correctness of these implementations.
"""
# load analyzer
base_dir = os.path.dirname(__file__)
relative_data_dir = 'data/ILSVRC2015_00078000'
data_dir = os.path.join(base_dir, relative_data_dir)
analyzer_file = os.path.join(data_dir, 'analyzer.p')
analyzer = Analyzer.load(analyzer_file, load_images_from_dir=False)
analyzer2 = Analyzer()
# check iteration
for frame_id, item in analyzer.items():
# check unpacking
prediction, ground_truth, image_path, image = analyzer.unpack_item(
item)[0:4]
# check __setitem by assigning to second analyzer
analyzer2[frame_id] = item
# add attributes to analyzer2
analyzer2.image_resize_factor = analyzer.image_resize_factor
analyzer2.video_processor = analyzer.video_processor
analyzer2.output_dir = analyzer.output_dir
analyzer2.class_names = analyzer.class_names
analyzer2.bbox_match_method = analyzer.bbox_match_method
analyzer2.iou_th = analyzer.iou_th
analyzer2.score_th = analyzer.score_th
# compare 2 analyzers
is_equal = compare_instances(analyzer, analyzer2)
assert is_equal
# check __delitem__
keys = list(analyzer.keys())
key_to_delete = keys[0]
del analyzer[key_to_delete]
import os
from analyze.analyzer import Analyzer
from analyze.viewer import AnalyzerViewer
if __name__ == '__main__':
import os
import panel as pn
pn.extension()
# load analyzer
base_dir = os.path.dirname(__file__)
relative_data_dir = '../tests/data/ILSVRC2015_00078000'
data_dir = os.path.join(base_dir, relative_data_dir)
analyzer_file = os.path.join(data_dir, 'analyzer.p')
analyzer = Analyzer.load(analyzer_file, load_images_from_dir=False)
os.chdir('..') # go one level up
# initialize viewer
viewer = AnalyzerViewer(analyzer, resize_factor=2.)
# view analyzer
viewer.view()
print('Done!')
def test_confusion_matrix():
base_dir = os.path.dirname(__file__)
relative_data_dir = 'data/ILSVRC2015_00078000'
data_dir = os.path.join(base_dir, relative_data_dir)
# load analyzer
analyzer_file = os.path.join(data_dir, 'analyzer.p')
analyzer = Analyzer.load(analyzer_file, load_images_from_dir=False)
# initialize total confusion matrix
num_classes = len(CLASS_NAMES)
cm_total = np.zeros((num_classes + 1, num_classes + 1))
# iterate over frames
for frame_id, item in analyzer.items():
# unpack data
prediction, ground_truth, image_path, image = analyzer.unpack_item(
item)[0:4]
# calculate confusion matrix of current frame
cm = ConfusionMatrix.calculate_confusion_matrix(prediction,
ground_truth,
CLASS_NAMES,
normalize=None,
score_th=0.3,
iou_th=0.5,
iou_criterion='all',
score_criterion='all',
display=False)
# analyze cm
# add current cm to total cm
cm_total += cm
# analyze cm_total
# normalize cm
cm_total_norm = ConfusionMatrix.normalize_confusion_matrix(cm_total,
norm_type='gt')
# save_fig_name = os.path.join(data_dir, 'analysis/confusion_matrix.png')
# ConfusionMatrix.plot_confusion_matrix(cm_total_norm, display_labels=CLASS_NAMES, save_fig_name=save_fig_name, display=False) # for display
cm_total_ref = np.array([
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 159., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 3.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 37., 0., 0., 0., 0., 0., 0., 0., 30.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.
],
[
0., 0., 0., 0., 0., 0., 0., 11., 0., 11., 0., 0., 0., 0., 0., 0.,
0., 0., 0., 0., 0., 0., 0., 11., 0., 0., 0., 0., 0., 0., 0., 0.
],
])
is_equal_mat = cm_total_ref == cm_total
is_equal = cm_total == pytest.approx(cm_total_ref)
assert is_equal
