def test_benchmark_text(self): self.benchmark_setUp() calibrator = Calibrator(Text('Text'), Image('all_shapes')) for calibration, random_starts in [(False, 0), (False, 1), (True, 0), (True, 1)]: finder = TextFinder() # the text backend has too many benchmarking combinations so let's restrict here finder.algorithms["threshold_filters2"] = ("adaptive",) finder.algorithms["threshold_filters3"] = ("adaptive",) finder.algorithms["threshold_filters2"] = ("adaptive",) finder.algorithms["threshold_filters3"] = ("adaptive",) # also get rid of these since they are not implemented anyway finder.algorithms["text_detectors"] = list(finder.algorithms["text_detectors"]) finder.algorithms["text_detectors"].remove("components") import cv2 # TODO: deprecate OpenCV 3.X versions after time if cv2.__version__.startswith("3."): finder.algorithms["text_detectors"].remove("east") finder.algorithms["text_recognizers"] = list(finder.algorithms["text_recognizers"]) finder.algorithms["text_recognizers"].remove("beamSearch") # one tesseract backend is enough for the unit test finder.algorithms["text_recognizers"].remove("tesseract") finder.algorithms["text_recognizers"].remove("pytesseract") results = calibrator.benchmark(finder, calibration=calibration, random_starts=random_starts) # pprint.pprint(results) self.assertGreater(len(results), 0, "There should be at least one benchmarked method") for result in results: self.assertIn("mixed", result[0], "Incorrect backend names for case '%s' %s %s" % result) self.assertGreaterEqual(result[1], 0.0, "Incorrect similarity for case '%s' %s %s" % result) self.assertLessEqual(result[1], 1.0, "Incorrect similarity for case '%s' %s %s" % result) self.assertGreater(result[2], 0.0, "Strictly positive time is required to run case '%s' %s %s" % result)
def calibration_setUp(self, needle, haystack, calibrate_backends): # use a single finder type for these tests finder = FeatureFinder() for category in calibrate_backends: finder.can_calibrate(category, True) calibrator = Calibrator(Image(needle), Image(haystack)) return calibrator.calibrate(finder)
def test_benchmark_cascade(self): self.benchmark_setUp() calibrator = Calibrator(Pattern('shape_blue_circle.xml'), Image('all_shapes')) for calibration, random_starts in [(False, 0), (False, 1), (True, 0), (True, 1)]: results = calibrator.benchmark(CascadeFinder(), calibration=calibration, random_starts=random_starts) # pprint.pprint(results) self.assertGreater(len(results), 0, "There should be at least one benchmarked method") for result in results: self.assertEqual(result[0], "", "Incorrect backend names for case '%s' %s %s" % result) # similarity is not available in the cascade backend self.assertEqual(result[1], 0.0, "Incorrect similarity for case '%s' %s %s" % result) self.assertGreater(result[2], 0.0, "Strictly positive time is required to run case '%s' %s %s" % result)
def test_benchmark_feature(self): self.benchmark_setUp() calibrator = Calibrator(Image('n_ibs'), Image('n_ibs')) for calibration, random_starts in [(False, 0), (False, 1), (True, 0), (True, 1)]: results = calibrator.benchmark(FeatureFinder(), calibration=calibration, random_starts=random_starts) # pprint.pprint(results) self.assertGreater(len(results), 0, "There should be at least one benchmarked method") for result in results: self.assertIn("mixed", result[0], "Incorrect backend names for case '%s' %s %s" % result) self.assertGreaterEqual(result[1], 0.0, "Incorrect similarity for case '%s' %s %s" % result) self.assertLessEqual(result[1], 1.0, "Incorrect similarity for case '%s' %s %s" % result) self.assertGreater(result[2], 0.0, "Strictly positive time is required to run case '%s' %s %s" % result)
def test_benchmark_template(self): self.benchmark_setUp() calibrator = Calibrator(Image('shape_blue_circle'), Image('all_shapes')) for calibration, random_starts in [(False, 0), (False, 1), (True, 0), (True, 1)]: results = calibrator.benchmark(TemplateFinder(), calibration=calibration, random_starts=random_starts) # pprint.pprint(results) self.assertGreater(len(results), 0, "There should be at least one benchmarked method") for result in results: # only normed backends are supported self.assertIn("_normed", result[0], "Incorrect backend names for case '%s' %s %s" % result) self.assertEqual(result[1], 1.0, "Incorrect similarity for case '%s' %s %s" % result) self.assertGreater(result[2], 0.0, "Strictly positive time is required to run case '%s' %s %s" % result)
def test_benchmark_contour(self): self.benchmark_setUp() # matching all shapes will require a modification of the minArea parameter calibrator = Calibrator(Image('shape_blue_circle'), Image('shape_blue_circle')) for calibration, random_starts in [(False, 0), (False, 1), (True, 0), (True, 1)]: results = calibrator.benchmark(ContourFinder(), calibration=calibration, random_starts=random_starts) # pprint.pprint(results) self.assertGreater(len(results), 0, "There should be at least one benchmarked method") for result in results: self.assertIn("mixed", result[0], "Incorrect backend names for case '%s' %s %s" % result) self.assertEqual(result[1], 1.0, "Incorrect similarity for case '%s' %s %s" % result) self.assertGreater(result[2], 0.0, "Strictly positive time is required to run case '%s' %s %s" % result)
def test_benchmark_deep(self): self.benchmark_setUp() calibrator = Calibrator(Pattern('cat'), Image('coco_cat')) for calibration, random_starts in [(False, 0), (False, 1), (True, 0), (True, 1)]: finder = DeepFinder() # get rid of backends that are not implemented anyway finder.algorithms["deep_learners"] = list(finder.algorithms["deep_learners"]) finder.algorithms["deep_learners"].remove("tensorflow") results = calibrator.benchmark(finder, calibration=calibration, random_starts=random_starts) # pprint.pprint(results) self.assertGreater(len(results), 0, "There should be at least one benchmarked method") for result in results: self.assertEqual("pytorch", result[0], "Incorrect backend names for case '%s' %s %s" % result) # TODO: the needle is found but with very low similarity - possibly due to different haystack size #self.assertEqual(result[1], 1.0, "Incorrect similarity for case '%s' %s %s" % result) self.assertGreater(result[2], 0.0, "Strictly positive time is required to run case '%s' %s %s" % result)
# non-default initial conditions for the calibration of various finder types #finder.configure(text_detector="contours") #finder.configure_backend(backend="sqdiff_normed", category="template") #finder.params["find"]["similarity"].value = 0.7 #finder.params["tempfeat"]["front_similarity"].value = 0.5 #finder.params["feature"]["ransacReprojThreshold"].value = 25.0 #finder.params["fdetect"]["MaxFeatures"].value = 10 #finder.params["text"]["datapath"].value = "../../misc" #finder.params["ocr"]["oem"].value = 0 #finder.params["tdetect"]["verticalVariance"].value = 5 #finder.params["threshold"]["blockSize"].value = 3 # synchronize at this stage to take into account all configuration finder.synchronize() # Main steps: calibration, searching, and benchmarking calibrator = Calibrator(NEEDLE, HAYSTACK) # uncomment this to use a list of needles and haystacks instead #calibrator = Calibrator(config="pairs.list") calibrator.run = calibrator.run_performance # uncomment these for alternative run functions #calibrator.run = calibrator.run_default #calibrator.run = calibrator.run_peak similarity_before = calibrator.calibrate(finder, max_attempts=1) # categories to calibrate for category in ENABLED: finder.can_calibrate(category, True) # example parameter to solo allow for calibration: #finder.params["threshold2"]["blockSize"].fixed = False similarity_after = calibrator.calibrate(finder, max_attempts=MAX_ATTEMPTS, max_exec_time=MAX_EXEC_TIME)
def list_setUp(self): with open("pairs.list", "w") as f: f.write("n_ibs" + " " + "h_ibs_viewport" + " max" + "\n") f.write("n_ibs" + " " + "h_ibs_rotated" + " max" + "\n") f.write("n_ibs" + " " + "h_ibs_scaled" + " max" + "\n") return Calibrator(config="pairs.list")