Esempio n. 1
0
 def load_config_file(filename='config_resource.json'):
     Logger.write('Loading config file:', filename)
     with open(filename, 'r', encoding='utf8') as file:
         config_json = json.load(file)
         Logger.write('Load success, content: \n' + json.dumps(config_json))
         file.close()
     return config_json
Esempio n. 2
0
 def boot(self, links):
     for rootlink in links:
         self.rootlink_queue.put(rootlink)
         Logger.write('Put rootlink:', rootlink)
         self.rootlink_queue.join()
         self.sublink_queue.join()
         self.image_queue.join()
Esempio n. 3
0
class E2ERunner(object):
    def __init__(self, config={}, globalConfig={}):
        self.config = Configuration(config)
        self.globalConfig = Configuration(globalConfig)
        self._parse_config()
        self.logger = Logger()
        self.config()

    def _parse_config(self):
        self._parse_blocks(self.config["blocks"])
        self.viz = self._parse_visualizer(self.config.default("viz", None))
        self.gtprov = self._parse_gt(self.config.default("gt", None))
        self.evals = self._parse_evals(self.config.default('eval', []))

    def _parse_blocks(self, blocks):
        self.blocks = [
            self._parse_block(block) for block in blocks
            if "disabled" not in block or not block["disabled"]
        ]

    def _parse_block(self, block):
        if block["type"] == "TextSeparation":
            return TextSeparation(self.globalConfig, block)
        elif block["type"] == "WordDetection":
            return WordDetection(block)
        elif block["type"] == "LineSegmentation":
            return LineSegmentation(block)
        elif block["type"] == "ParagraphDetection":
            return ParagraphDetection(block)
        elif block["type"] == "UnigramLanguageModel":
            return UnigramLanguageModel(block)
        elif block["type"] == "Ceiling":
            return Ceiling(block)
        elif block["type"] == "TranscriptionAndClassification":
            return TranscriptionAndClassification(self.globalConfig, block)

    def _parse_evals(self, eval_configs):
        return [self._parse_eval(config) for config in eval_configs]

    def _parse_eval(self, config):
        if config is None:
            return None
        if config["type"] == "IoU":
            return IoU(config)
        elif config["type"] == "IoUPixelSum":
            return IoUPixelSum(config)
        elif config["type"] == "BagOfWords":
            return BagOfWords(config)
        elif config["type"] == "IoUCER":
            return IoUCER(config)

    def _parse_data(self, data_config):
        if isinstance(data_config, list):
            return data_config
        else:
            prefix = data_config["prefix"] if "prefix" in data_config else ""
            filenames = list(
                filter(
                    lambda f: f.endswith(data_config["suffix"]) and f.
                    startswith(prefix), os.listdir(data_config["path"])))
            if data_config["limit"] > 0:
                filenames = filenames[:data_config["limit"]]
            return [
                os.path.join(data_config["path"], filename)
                for filename in filenames
            ]

    def _parse_visualizer(self, viz_config):
        if viz_config is None:
            return None
        if viz_config["type"] == "RegionVisualizer":
            return RegionVisualizer(viz_config)
        elif viz_config["type"] == "ImageVisualizer":
            return ImageVisualizer(viz_config)
        elif viz_config["type"] == "SeparatedVisualizer":
            return SeparatedVisualizer(viz_config)

    def _parse_gt(self, gt_config):
        if gt_config is None:
            return None
        if gt_config["type"] == "WordRegion":
            return WordRegionGTProvider()
        elif gt_config["type"] == "ParagraphRegion":
            return ParagraphRegionGTProvider()
        elif gt_config["type"] == "LineRegion":
            return LineRegionGTProvider()

    def __call__(self, log_prefix="E2E", skip_range_evaluation=False):
        if not skip_range_evaluation and self.config.default("ranger", False):
            self.logger.write("Entering Range Execution Mode")
            return self._range_exec()
        start = time()
        self.scores = {}
        data = self._parse_data(self.config["data"])
        results = []
        times = []
        for idx, file in enumerate(data):
            file_time = time()
            self.logger.progress(log_prefix, idx, len(data))
            results.append(self._exec(file))
            times.append(time() - file_time)
        [block.close() for block in self.blocks]
        if len(self.evals) > 0:
            final_scores = {
                "time": time() - start,
                "median time": np.median(times),
                "avg time": np.average(times)
            }
            for score_key in self.scores:
                final_scores[score_key] = np.average(self.scores[score_key])
            self.logger.summary(log_prefix, final_scores)
        return results

    def _get_range(self):
        if type(self.config["ranger.values"]) is dict:
            return frange(self.config["ranger.values.from"],
                          self.config["ranger.values.to"],
                          self.config["ranger.values.step"])

    def _range_exec(self):
        def set_config(value):
            for path in self.config.default(
                    "ranger.paths", [self.config.default("ranger.path", [])]):
                current = self.config
                for step in path[:-1]:
                    current = current[step]
                current[path[-1]] = value
            self._parse_config()

        for val in self._get_range():
            set_config(val)
            prefix = self.config.default("ranger.template", "value {}")
            self(log_prefix=prefix.format(val), skip_range_evaluation=True)

    def _exec(self, file):
        original = cv2.imread(file)
        last_output = original.copy()

        for block in self.blocks:
            last_output = block(last_output, file)
        res = {"file": file, "original": original, "result": last_output}
        if self.gtprov is not None:
            gt = self.gtprov(file, original)
        if self.viz is not None:
            vizimage = res["original"].copy()
            if self.gtprov is not None and self.config.default(
                    'gt.viz', False):
                vizimage = self.viz(vizimage, gt, True)
            if len(self.blocks) > 0:
                vizimage = self.viz(vizimage, res["result"], False)
            self.viz.store(vizimage, file)
            res["viz"] = vizimage
        if len(self.evals) > 0:
            for evl in self.evals:
                scores = evl(gt, res["result"])
                for score_key in scores.keys():
                    self.scores[score_key] = [
                        scores[score_key]
                    ] if score_key not in self.scores else [
                        scores[score_key], *self.scores[score_key]
                    ]
        return res