def main(args: EvalArgs):
# Local imports (imports that require tensorflow)
from calamari_ocr.ocr.scenario import CalamariScenario
from calamari_ocr.ocr.dataset.data import Data
from calamari_ocr.ocr.evaluator import Evaluator
if args.checkpoint:
saved_model = SavedCalamariModel(args.checkpoint, auto_update=True)
trainer_params = CalamariScenario.trainer_cls().params_cls().from_dict(saved_model.dict)
data_params = trainer_params.scenario.data
else:
data_params = Data.default_params()
data = Data(data_params)
pred_data = args.pred if args.pred is not None else args.gt.to_prediction()
evaluator = Evaluator(args.evaluator, data=data)
evaluator.preload_gt(gt_dataset=args.gt)
r = evaluator.run(gt_dataset=args.gt, pred_dataset=pred_data)
# TODO: More output
print("Evaluation result")
print("=================")
print("")
print(
"Got mean normalized label error rate of {:.2%} ({} errs, {} total chars, {} sync errs)".format(
r["avg_ler"], r["total_char_errs"], r["total_chars"], r["total_sync_errs"]
)
)
# sort descending
print_confusions(r, args.n_confusions)
samples = data.create_pipeline(evaluator.params.setup, args.gt).reader().samples()
print_worst_lines(r, samples, args.n_worst_lines)
if args.xlsx_output:
write_xlsx(
args.xlsx_output,
[
{
"prefix": "evaluation",
"results": r,
"gt_files": [s["id"] for s in samples],
}
],
)
return r
def main(args=None):
parser = PAIArgumentParser()
parser.add_argument('--version', action='version', version='%(prog)s v' + __version__)
parser.add_argument("--n_cols", type=int, default=1)
parser.add_argument("--n_rows", type=int, default=5)
parser.add_argument("--select", type=int, nargs="+", default=[])
parser.add_argument("--preload", action='store_true', help='Simulate preloading')
parser.add_argument("--as_validation", action='store_true', help="Access as validation instead of training data.")
parser.add_argument("--n_augmentations", type=float, default=0)
parser.add_argument("--no_plot", action='store_true', help='This parameter is for testing only')
parser.add_root_argument("data", DataWrapper)
args = parser.parse_args(args=args)
data_wrapper: DataWrapper = args.data
data_params = data_wrapper.data
data_params.pre_proc.run_parallel = False
data_params.pre_proc.erase_all(PrepareSampleProcessorParams)
for p in data_params.pre_proc.processors_of_type(AugmentationProcessorParams):
p.n_augmentations = args.n_augmentations
data_params.__post_init__()
data_wrapper.pipeline.mode = PipelineMode.EVALUATION if args.as_validation else PipelineMode.TRAINING
data_wrapper.gen.prepare_for_mode(data_wrapper.pipeline.mode)
data = Data(data_params)
if len(args.select) == 0:
args.select = list(range(len(data_wrapper.gen)))
else:
try:
data_wrapper.gen.select(args.select)
except NotImplementedError:
logger.warning(f"Selecting is not supported for a data generator of type {type(data_wrapper.gen)}. "
f"Resuming without selection.")
data_pipeline = data.create_pipeline(data_wrapper.pipeline, data_wrapper.gen)
if args.preload:
data_pipeline = data_pipeline.as_preloaded()
if args.no_plot:
with data_pipeline as dataset:
list(zip(args.select, dataset.generate_input_samples(auto_repeat=False)))
return
import matplotlib.pyplot as plt
f, ax = plt.subplots(args.n_rows, args.n_cols, sharey='all')
row, col = 0, 0
with data_pipeline as dataset:
for i, (id, sample) in enumerate(zip(args.select, dataset.generate_input_samples(auto_repeat=False))):
line, text, params = sample.inputs, sample.targets, sample.meta
if args.n_cols == 1:
ax[row].imshow(line.transpose())
ax[row].set_title("ID: {}\n{}".format(id, text))
else:
ax[row, col].imshow(line.transpose())
ax[row, col].set_title("ID: {}\n{}".format(id, text))
row += 1
if row == args.n_rows:
row = 0
col += 1
if col == args.n_cols or i == len(dataset) - 1:
plt.show()
f, ax = plt.subplots(args.n_rows, args.n_cols, sharey='all')
row, col = 0, 0
plt.show()
def main():
# Local imports (imports that require tensorflow)
from calamari_ocr.ocr.scenario import Scenario
from calamari_ocr.ocr.dataset.data import Data
from calamari_ocr.ocr.evaluator import Evaluator
parser = ArgumentParser()
parser.add_argument("--dataset", type=DataSetType.from_string, choices=list(DataSetType), default=DataSetType.FILE)
parser.add_argument("--gt", nargs="+", required=True,
help="Ground truth files (.gt.txt extension). "
"Optionally, you can pass a single json file defining all parameters.")
parser.add_argument("--pred", nargs="+", default=None,
help="Prediction files if provided. Else files with .pred.txt are expected at the same "
"location as the gt.")
parser.add_argument("--pred_dataset", type=DataSetType.from_string, choices=list(DataSetType), default=DataSetType.FILE)
parser.add_argument("--pred_ext", type=str, default=".pred.txt",
help="Extension of the predicted text files")
parser.add_argument("--n_confusions", type=int, default=10,
help="Only print n most common confusions. Defaults to 10, use -1 for all.")
parser.add_argument("--n_worst_lines", type=int, default=0,
help="Print the n worst recognized text lines with its error")
parser.add_argument("--xlsx_output", type=str,
help="Optionally write a xlsx file with the evaluation results")
parser.add_argument("--num_threads", type=int, default=1,
help="Number of threads to use for evaluation")
parser.add_argument("--non_existing_file_handling_mode", type=str, default="error",
help="How to handle non existing .pred.txt files. Possible modes: skip, empty, error. "
"'Skip' will simply skip the evaluation of that file (not counting it to errors). "
"'Empty' will handle this file as would it be empty (fully checking for errors)."
"'Error' will throw an exception if a file is not existing. This is the default behaviour.")
parser.add_argument("--skip_empty_gt", action="store_true", default=False,
help="Ignore lines of the gt that are empty.")
parser.add_argument("--no_progress_bars", action="store_true",
help="Do not show any progress bars")
parser.add_argument("--checkpoint", type=str, default=None,
help="Specify an optional checkpoint to parse the text preprocessor (for the gt txt files)")
# page xml specific args
parser.add_argument("--pagexml_gt_text_index", default=0)
parser.add_argument("--pagexml_pred_text_index", default=1)
args = parser.parse_args()
# check if loading a json file
if len(args.gt) == 1 and args.gt[0].endswith("json"):
with open(args.gt[0], 'r') as f:
json_args = json.load(f)
for key, value in json_args.items():
setattr(args, key, value)
logger.info("Resolving files")
gt_files = sorted(glob_all(args.gt))
if args.pred:
pred_files = sorted(glob_all(args.pred))
else:
pred_files = [split_all_ext(gt)[0] + args.pred_ext for gt in gt_files]
args.pred_dataset = args.dataset
if args.non_existing_file_handling_mode.lower() == "skip":
non_existing_pred = [p for p in pred_files if not os.path.exists(p)]
for f in non_existing_pred:
idx = pred_files.index(f)
del pred_files[idx]
del gt_files[idx]
data_params = Data.get_default_params()
if args.checkpoint:
saved_model = SavedCalamariModel(args.checkpoint, auto_update=True)
trainer_params = Scenario.trainer_params_from_dict(saved_model.dict)
data_params = trainer_params.scenario_params.data_params
data = Data(data_params)
gt_reader_args = FileDataReaderArgs(
text_index=args.pagexml_gt_text_index
)
pred_reader_args = FileDataReaderArgs(
text_index=args.pagexml_pred_text_index
)
gt_data_set = PipelineParams(
type=args.dataset,
text_files=gt_files,
data_reader_args=gt_reader_args,
skip_invalid=args.skip_empty_gt,
)
pred_data_set = PipelineParams(
type=args.pred_dataset,
text_files=pred_files,
data_reader_args=pred_reader_args,
)
evaluator = Evaluator(data=data)
evaluator.preload_gt(gt_dataset=gt_data_set)
r = evaluator.run(gt_dataset=gt_data_set, pred_dataset=pred_data_set, processes=args.num_threads,
progress_bar=not args.no_progress_bars)
# TODO: More output
print("Evaluation result")
print("=================")
print("")
print("Got mean normalized label error rate of {:.2%} ({} errs, {} total chars, {} sync errs)".format(
r["avg_ler"], r["total_char_errs"], r["total_chars"], r["total_sync_errs"]))
# sort descending
print_confusions(r, args.n_confusions)
print_worst_lines(r, data.create_pipeline(PipelineMode.Targets, gt_data_set).reader().samples(), args.n_worst_lines)
if args.xlsx_output:
write_xlsx(args.xlsx_output,
[{
"prefix": "evaluation",
"results": r,
"gt_files": gt_files,
}])