def load_config_file(filename='config_resource.json'):
Logger.write('Loading config file:', filename)
with open(filename, 'r', encoding='utf8') as file:
config_json = json.load(file)
Logger.write('Load success, content: \n' + json.dumps(config_json))
file.close()
return config_json
def boot(self, links):
for rootlink in links:
self.rootlink_queue.put(rootlink)
Logger.write('Put rootlink:', rootlink)
self.rootlink_queue.join()
self.sublink_queue.join()
self.image_queue.join()
class E2ERunner(object):
def __init__(self, config={}, globalConfig={}):
self.config = Configuration(config)
self.globalConfig = Configuration(globalConfig)
self._parse_config()
self.logger = Logger()
self.config()
def _parse_config(self):
self._parse_blocks(self.config["blocks"])
self.viz = self._parse_visualizer(self.config.default("viz", None))
self.gtprov = self._parse_gt(self.config.default("gt", None))
self.evals = self._parse_evals(self.config.default('eval', []))
def _parse_blocks(self, blocks):
self.blocks = [
self._parse_block(block) for block in blocks
if "disabled" not in block or not block["disabled"]
]
def _parse_block(self, block):
if block["type"] == "TextSeparation":
return TextSeparation(self.globalConfig, block)
elif block["type"] == "WordDetection":
return WordDetection(block)
elif block["type"] == "LineSegmentation":
return LineSegmentation(block)
elif block["type"] == "ParagraphDetection":
return ParagraphDetection(block)
elif block["type"] == "UnigramLanguageModel":
return UnigramLanguageModel(block)
elif block["type"] == "Ceiling":
return Ceiling(block)
elif block["type"] == "TranscriptionAndClassification":
return TranscriptionAndClassification(self.globalConfig, block)
def _parse_evals(self, eval_configs):
return [self._parse_eval(config) for config in eval_configs]
def _parse_eval(self, config):
if config is None:
return None
if config["type"] == "IoU":
return IoU(config)
elif config["type"] == "IoUPixelSum":
return IoUPixelSum(config)
elif config["type"] == "BagOfWords":
return BagOfWords(config)
elif config["type"] == "IoUCER":
return IoUCER(config)
def _parse_data(self, data_config):
if isinstance(data_config, list):
return data_config
else:
prefix = data_config["prefix"] if "prefix" in data_config else ""
filenames = list(
filter(
lambda f: f.endswith(data_config["suffix"]) and f.
startswith(prefix), os.listdir(data_config["path"])))
if data_config["limit"] > 0:
filenames = filenames[:data_config["limit"]]
return [
os.path.join(data_config["path"], filename)
for filename in filenames
]
def _parse_visualizer(self, viz_config):
if viz_config is None:
return None
if viz_config["type"] == "RegionVisualizer":
return RegionVisualizer(viz_config)
elif viz_config["type"] == "ImageVisualizer":
return ImageVisualizer(viz_config)
elif viz_config["type"] == "SeparatedVisualizer":
return SeparatedVisualizer(viz_config)
def _parse_gt(self, gt_config):
if gt_config is None:
return None
if gt_config["type"] == "WordRegion":
return WordRegionGTProvider()
elif gt_config["type"] == "ParagraphRegion":
return ParagraphRegionGTProvider()
elif gt_config["type"] == "LineRegion":
return LineRegionGTProvider()
def __call__(self, log_prefix="E2E", skip_range_evaluation=False):
if not skip_range_evaluation and self.config.default("ranger", False):
self.logger.write("Entering Range Execution Mode")
return self._range_exec()
start = time()
self.scores = {}
data = self._parse_data(self.config["data"])
results = []
times = []
for idx, file in enumerate(data):
file_time = time()
self.logger.progress(log_prefix, idx, len(data))
results.append(self._exec(file))
times.append(time() - file_time)
[block.close() for block in self.blocks]
if len(self.evals) > 0:
final_scores = {
"time": time() - start,
"median time": np.median(times),
"avg time": np.average(times)
}
for score_key in self.scores:
final_scores[score_key] = np.average(self.scores[score_key])
self.logger.summary(log_prefix, final_scores)
return results
def _get_range(self):
if type(self.config["ranger.values"]) is dict:
return frange(self.config["ranger.values.from"],
self.config["ranger.values.to"],
self.config["ranger.values.step"])
def _range_exec(self):
def set_config(value):
for path in self.config.default(
"ranger.paths", [self.config.default("ranger.path", [])]):
current = self.config
for step in path[:-1]:
current = current[step]
current[path[-1]] = value
self._parse_config()
for val in self._get_range():
set_config(val)
prefix = self.config.default("ranger.template", "value {}")
self(log_prefix=prefix.format(val), skip_range_evaluation=True)
def _exec(self, file):
original = cv2.imread(file)
last_output = original.copy()
for block in self.blocks:
last_output = block(last_output, file)
res = {"file": file, "original": original, "result": last_output}
if self.gtprov is not None:
gt = self.gtprov(file, original)
if self.viz is not None:
vizimage = res["original"].copy()
if self.gtprov is not None and self.config.default(
'gt.viz', False):
vizimage = self.viz(vizimage, gt, True)
if len(self.blocks) > 0:
vizimage = self.viz(vizimage, res["result"], False)
self.viz.store(vizimage, file)
res["viz"] = vizimage
if len(self.evals) > 0:
for evl in self.evals:
scores = evl(gt, res["result"])
for score_key in scores.keys():
self.scores[score_key] = [
scores[score_key]
] if score_key not in self.scores else [
scores[score_key], *self.scores[score_key]
]
return res
