def test_load_and_save(): """ Load, save and load again full and thin analyzer instances, and verify that content is identical. """ base_dir = os.path.dirname(__file__) relative_data_dir = 'data/1_example' data_dir = os.path.join(base_dir, relative_data_dir) # ---------------------------- # load and save full analyzer # ---------------------------- # load analyzer analyzer_file = os.path.join(data_dir, 'analyzer_full.p') analyzer_full = Analyzer.load(analyzer_file) # save analyzer in temporary directory outupt_dir = os.path.join(data_dir, 'tmp_full') shutil.rmtree(outupt_dir, ignore_errors=True) # delete existing temp dir # save instance with image dir output_file_full = os.path.join(outupt_dir, 'analyzer.p') analyzer_full.save(output_file_full, save_thin_instance=False, save_images_in_dir=True, image_name_template='{:08d}.png') # ---------------------------- # load and save thin analyzer # ---------------------------- analyzer_file = os.path.join(data_dir, 'analyzer_thin.p') analyzer_thin = Analyzer.load(analyzer_file, load_images_from_dir=True, sfx='png') # use lossless png image, not jpg # save analyzer in temporary directory outupt_dir = os.path.join(data_dir, 'tmp_thin') shutil.rmtree(outupt_dir, ignore_errors=True) # delete existing temp dir # save instance with image dir output_file_thin = os.path.join(outupt_dir, 'analyzer.p') analyzer_thin.save(output_file_thin, save_thin_instance=True, save_images_in_dir=True, image_name_template='{:08d}.png') # use lossless png image, not jpg # ---------------------------------------------------------------- # load analyzers from saved dirs and compare content thin analyzer # ---------------------------------------------------------------- # load analyzers analyzer_full2 = Analyzer.load(output_file_full) analyzer_thin2 = Analyzer.load(output_file_thin, load_images_from_dir=True, sfx='png') # use lossless png image, not jpg # compare content is_equal = compare_instances(analyzer_full2, analyzer_thin2) assert is_equal
def load_test_analyzer(data_dir='ILSVRC2015_00078000'): """ Load analyzer object from saved test data. Parameters ---------- data_dir : str, optional Data directory name, should be name of one of the folders inside analyze/tests/data/. Returns ------- anaylzer : Analyzer Analyzer object. analyzer_root_dir : str Path of analyzer root directory, such that full images path is the concatenation of analyzer_root_dir and analyzer.data image_path. """ # load reference analyzer base_dir = os.path.dirname(__file__) relative_data_dir = os.path.join('data', data_dir) data_dir = os.path.join(base_dir, relative_data_dir) analyzer_file = os.path.join(data_dir, 'analyzer.p') analyzer = Analyzer.load(analyzer_file, load_images_from_dir=False) analyzer_root_dir = os.path.join(base_dir, '..') return analyzer, analyzer_root_dir
def test_visualize_example(): base_dir = os.path.dirname(__file__) relative_data_dir = 'data/ILSVRC2015_00078000' data_dir = os.path.join(base_dir, relative_data_dir) analyze_root_dir = os.path.abspath(os.path.join(base_dir, '..')) os.chdir(analyze_root_dir) # load analyzer analyzer_file = os.path.join(data_dir, 'analyzer.p') analyzer = Analyzer.load(analyzer_file, load_images_from_dir=True) # visualize example save_fig_path = os.path.join(base_dir, 'save_fig_example.png') frame_id = 40 display = False # True image = analyzer.visualize_example(key=frame_id, class_names=CLASS_NAMES, display=display, save_fig_path=save_fig_path) # compare to reference image image_ref_path = os.path.join( base_dir, 'data/visualizations/visualization_example.png') image_ref = cv2.imread(image_ref_path, cv2.IMREAD_UNCHANGED) is_close = np.isclose(image, image_ref, atol=2) # allow up to 2 gray level difference is_all_close = np.all(is_close) assert is_all_close
def test_empty_preds_gts(): base_dir = os.path.dirname(__file__) relative_data_dir = 'data/ILSVRC2015_00078000' data_dir = os.path.join(base_dir, relative_data_dir) # load analyzer analyzer_file = os.path.join(data_dir, 'analyzer.p') analyzer = Analyzer.load(analyzer_file, load_images_from_dir=False) # initialize total confusion matrix num_classes = len(CLASS_NAMES) cm_total = np.zeros((num_classes + 1, num_classes + 1)) # iterate over frames counter = 0 for frame_id, item in analyzer.items(): # unpack data prediction, ground_truth, image_path, image = analyzer.unpack_item( item)[0:4] if counter == 1: # delete predictions prediction = Box([], image_shape=(100, 200)) elif counter == 2: # delete groud truths ground_truth = Box([], image_shape=(100, 200)) elif counter == 3: # delete image shape try: ground_truth = Box([], image_shape=None) except ValueError as e: cond3 = e.__repr__( ) == "ValueError('image_shape must be tuple of length 2 (height, width) or 3 (height, width, channels), got None',)" # calculate confusion matrix of current frame cm = ConfusionMatrix.calculate_confusion_matrix(prediction, ground_truth, CLASS_NAMES, normalize=None, score_th=0.3, iou_th=0.5, iou_criterion='all', score_criterion='all', display=False) if counter == 1: # delete predictions cond1 = cm[:, -1].sum() == 3 elif counter == 2: # delete groud truths cond2 = cm[-1, :].sum() == 3 # analyze cm # advance counter counter += 1 # check conditions assert cond1 assert cond2 assert cond3
def test_initialize_with_simple_wrapper(): base_dir = os.path.dirname(__file__) relative_data_dir = 'data/ILSVRC2015_00078000' data_dir = os.path.join(base_dir, relative_data_dir) # load analyzer analyzer_file = os.path.join(data_dir, 'analyzer.p') analyzer = Analyzer.load(analyzer_file, load_images_from_dir=False) # initialize total confusion matrix num_classes = len(CLASS_NAMES) cm_total = np.zeros((num_classes + 1, num_classes + 1))
def generate_report(): # load reference analyzer base_dir = os.path.dirname(__file__) relative_data_dir = '../tests/data/ILSVRC2015_00078000' data_dir = os.path.join(base_dir, relative_data_dir) analyzer_file = os.path.join(data_dir, 'analyzer.p') analyzer = Analyzer.load(analyzer_file, load_images_from_dir=False) # set output directory analyzer.output_dir = os.path.join(base_dir, 'output', 'generate_report') os.makedirs(analyzer.output_dir, exist_ok=True) # generate report analyzer.evaluate_performance(generate_report=True) pass
def test_analyzer_mutable_mapping_implementation(): """ Analyzer class inherits from collections.abc.MutableMapping, which demands implementation of several class methods such as __setitem__, __iter__, etc. Here we will check the correctness of these implementations. """ # load analyzer base_dir = os.path.dirname(__file__) relative_data_dir = 'data/ILSVRC2015_00078000' data_dir = os.path.join(base_dir, relative_data_dir) analyzer_file = os.path.join(data_dir, 'analyzer.p') analyzer = Analyzer.load(analyzer_file, load_images_from_dir=False) analyzer2 = Analyzer() # check iteration for frame_id, item in analyzer.items(): # check unpacking prediction, ground_truth, image_path, image = analyzer.unpack_item( item)[0:4] # check __setitem by assigning to second analyzer analyzer2[frame_id] = item # add attributes to analyzer2 analyzer2.image_resize_factor = analyzer.image_resize_factor analyzer2.video_processor = analyzer.video_processor analyzer2.output_dir = analyzer.output_dir analyzer2.class_names = analyzer.class_names analyzer2.bbox_match_method = analyzer.bbox_match_method analyzer2.iou_th = analyzer.iou_th analyzer2.score_th = analyzer.score_th # compare 2 analyzers is_equal = compare_instances(analyzer, analyzer2) assert is_equal # check __delitem__ keys = list(analyzer.keys()) key_to_delete = keys[0] del analyzer[key_to_delete]
import os from analyze.analyzer import Analyzer from analyze.viewer import AnalyzerViewer if __name__ == '__main__': import os import panel as pn pn.extension() # load analyzer base_dir = os.path.dirname(__file__) relative_data_dir = '../tests/data/ILSVRC2015_00078000' data_dir = os.path.join(base_dir, relative_data_dir) analyzer_file = os.path.join(data_dir, 'analyzer.p') analyzer = Analyzer.load(analyzer_file, load_images_from_dir=False) os.chdir('..') # go one level up # initialize viewer viewer = AnalyzerViewer(analyzer, resize_factor=2.) # view analyzer viewer.view() print('Done!')
def test_confusion_matrix(): base_dir = os.path.dirname(__file__) relative_data_dir = 'data/ILSVRC2015_00078000' data_dir = os.path.join(base_dir, relative_data_dir) # load analyzer analyzer_file = os.path.join(data_dir, 'analyzer.p') analyzer = Analyzer.load(analyzer_file, load_images_from_dir=False) # initialize total confusion matrix num_classes = len(CLASS_NAMES) cm_total = np.zeros((num_classes + 1, num_classes + 1)) # iterate over frames for frame_id, item in analyzer.items(): # unpack data prediction, ground_truth, image_path, image = analyzer.unpack_item( item)[0:4] # calculate confusion matrix of current frame cm = ConfusionMatrix.calculate_confusion_matrix(prediction, ground_truth, CLASS_NAMES, normalize=None, score_th=0.3, iou_th=0.5, iou_criterion='all', score_criterion='all', display=False) # analyze cm # add current cm to total cm cm_total += cm # analyze cm_total # normalize cm cm_total_norm = ConfusionMatrix.normalize_confusion_matrix(cm_total, norm_type='gt') # save_fig_name = os.path.join(data_dir, 'analysis/confusion_matrix.png') # ConfusionMatrix.plot_confusion_matrix(cm_total_norm, display_labels=CLASS_NAMES, save_fig_name=save_fig_name, display=False) # for display cm_total_ref = np.array([ [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 159., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 3. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 37., 0., 0., 0., 0., 0., 0., 0., 30. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0. ], [ 0., 0., 0., 0., 0., 0., 0., 11., 0., 11., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 11., 0., 0., 0., 0., 0., 0., 0., 0. ], ]) is_equal_mat = cm_total_ref == cm_total is_equal = cm_total == pytest.approx(cm_total_ref) assert is_equal