def test_benchmark_text(self): self.benchmark_setUp() calibrator = Calibrator(Text('Text'), Image('all_shapes')) for calibration, random_starts in [(False, 0), (False, 1), (True, 0), (True, 1)]: finder = TextFinder() # the text backend has too many benchmarking combinations so let's restrict here finder.algorithms["threshold_filters2"] = ("adaptive",) finder.algorithms["threshold_filters3"] = ("adaptive",) finder.algorithms["threshold_filters2"] = ("adaptive",) finder.algorithms["threshold_filters3"] = ("adaptive",) # also get rid of these since they are not implemented anyway finder.algorithms["text_detectors"] = list(finder.algorithms["text_detectors"]) finder.algorithms["text_detectors"].remove("components") import cv2 # TODO: deprecate OpenCV 3.X versions after time if cv2.__version__.startswith("3."): finder.algorithms["text_detectors"].remove("east") finder.algorithms["text_recognizers"] = list(finder.algorithms["text_recognizers"]) finder.algorithms["text_recognizers"].remove("beamSearch") # one tesseract backend is enough for the unit test finder.algorithms["text_recognizers"].remove("tesseract") finder.algorithms["text_recognizers"].remove("pytesseract") results = calibrator.benchmark(finder, calibration=calibration, random_starts=random_starts) # pprint.pprint(results) self.assertGreater(len(results), 0, "There should be at least one benchmarked method") for result in results: self.assertIn("mixed", result[0], "Incorrect backend names for case '%s' %s %s" % result) self.assertGreaterEqual(result[1], 0.0, "Incorrect similarity for case '%s' %s %s" % result) self.assertLessEqual(result[1], 1.0, "Incorrect similarity for case '%s' %s %s" % result) self.assertGreater(result[2], 0.0, "Strictly positive time is required to run case '%s' %s %s" % result)
def test_benchmark_cascade(self): self.benchmark_setUp() calibrator = Calibrator(Pattern('shape_blue_circle.xml'), Image('all_shapes')) for calibration, random_starts in [(False, 0), (False, 1), (True, 0), (True, 1)]: results = calibrator.benchmark(CascadeFinder(), calibration=calibration, random_starts=random_starts) # pprint.pprint(results) self.assertGreater(len(results), 0, "There should be at least one benchmarked method") for result in results: self.assertEqual(result[0], "", "Incorrect backend names for case '%s' %s %s" % result) # similarity is not available in the cascade backend self.assertEqual(result[1], 0.0, "Incorrect similarity for case '%s' %s %s" % result) self.assertGreater(result[2], 0.0, "Strictly positive time is required to run case '%s' %s %s" % result)
def test_benchmark_feature(self): self.benchmark_setUp() calibrator = Calibrator(Image('n_ibs'), Image('n_ibs')) for calibration, random_starts in [(False, 0), (False, 1), (True, 0), (True, 1)]: results = calibrator.benchmark(FeatureFinder(), calibration=calibration, random_starts=random_starts) # pprint.pprint(results) self.assertGreater(len(results), 0, "There should be at least one benchmarked method") for result in results: self.assertIn("mixed", result[0], "Incorrect backend names for case '%s' %s %s" % result) self.assertGreaterEqual(result[1], 0.0, "Incorrect similarity for case '%s' %s %s" % result) self.assertLessEqual(result[1], 1.0, "Incorrect similarity for case '%s' %s %s" % result) self.assertGreater(result[2], 0.0, "Strictly positive time is required to run case '%s' %s %s" % result)
def test_benchmark_template(self): self.benchmark_setUp() calibrator = Calibrator(Image('shape_blue_circle'), Image('all_shapes')) for calibration, random_starts in [(False, 0), (False, 1), (True, 0), (True, 1)]: results = calibrator.benchmark(TemplateFinder(), calibration=calibration, random_starts=random_starts) # pprint.pprint(results) self.assertGreater(len(results), 0, "There should be at least one benchmarked method") for result in results: # only normed backends are supported self.assertIn("_normed", result[0], "Incorrect backend names for case '%s' %s %s" % result) self.assertEqual(result[1], 1.0, "Incorrect similarity for case '%s' %s %s" % result) self.assertGreater(result[2], 0.0, "Strictly positive time is required to run case '%s' %s %s" % result)
def test_benchmark_contour(self): self.benchmark_setUp() # matching all shapes will require a modification of the minArea parameter calibrator = Calibrator(Image('shape_blue_circle'), Image('shape_blue_circle')) for calibration, random_starts in [(False, 0), (False, 1), (True, 0), (True, 1)]: results = calibrator.benchmark(ContourFinder(), calibration=calibration, random_starts=random_starts) # pprint.pprint(results) self.assertGreater(len(results), 0, "There should be at least one benchmarked method") for result in results: self.assertIn("mixed", result[0], "Incorrect backend names for case '%s' %s %s" % result) self.assertEqual(result[1], 1.0, "Incorrect similarity for case '%s' %s %s" % result) self.assertGreater(result[2], 0.0, "Strictly positive time is required to run case '%s' %s %s" % result)
def test_benchmark_deep(self): self.benchmark_setUp() calibrator = Calibrator(Pattern('cat'), Image('coco_cat')) for calibration, random_starts in [(False, 0), (False, 1), (True, 0), (True, 1)]: finder = DeepFinder() # get rid of backends that are not implemented anyway finder.algorithms["deep_learners"] = list(finder.algorithms["deep_learners"]) finder.algorithms["deep_learners"].remove("tensorflow") results = calibrator.benchmark(finder, calibration=calibration, random_starts=random_starts) # pprint.pprint(results) self.assertGreater(len(results), 0, "There should be at least one benchmarked method") for result in results: self.assertEqual("pytorch", result[0], "Incorrect backend names for case '%s' %s %s" % result) # TODO: the needle is found but with very low similarity - possibly due to different haystack size #self.assertEqual(result[1], 1.0, "Incorrect similarity for case '%s' %s %s" % result) self.assertGreater(result[2], 0.0, "Strictly positive time is required to run case '%s' %s %s" % result)
#finder.params["threshold2"]["blockSize"].fixed = False similarity_after = calibrator.calibrate(finder, max_attempts=MAX_ATTEMPTS, max_exec_time=MAX_EXEC_TIME) logging.info("Similarity before and after calibration: %s -> %s", similarity_before, similarity_after) similarity_global = calibrator.search(finder, random_starts=RANDOM_STARTS, uniform=UNIFORM_DRAW, calibration=CALIBRATED_SEARCH, max_attempts=MAX_ATTEMPTS, max_exec_time=MAX_EXEC_TIME) logging.info("Similarity after search (Monte Carlo calibration): %s -> %s", similarity_before, similarity_global) logging.info("Best found parameters:\n%s", pprint.pformat(finder.params)) results = calibrator.benchmark(finder, random_starts=BENCHMARK_RANDOM_STARTS, uniform=UNIFORM_DRAW, calibration=CALIBRATED_BENCHMARK, max_attempts=MAX_ATTEMPTS, max_exec_time=MAX_EXEC_TIME) logging.info("Benchmarking results (method, similarity, location, time):\n%s", pprint.pformat(results)) # Final cleanup steps if REMOVE_LOGPATH: shutil.rmtree(LOGPATH) GlobalConfig.image_logging_level = logging.ERROR GlobalConfig.image_logging_destination = "./imglog" GlobalConfig.image_logging_step_width = 3