def test_raw_prediction_voted(self):
args = PredictionAttrs()
predictor = MultiPredictor(checkpoints=args.checkpoint)
images = [np.array(Image.open(file), dtype=np.uint8) for file in args.files]
for file, image in zip(args.files, images):
r = list(predictor.predict_raw([image], progress_bar=False))[0]
print(file, [rn.sentence for rn in r])
class CalamariRecognize(Processor):
def __init__(self, *args, **kwargs):
kwargs['ocrd_tool'] = OCRD_TOOL['tools'][TOOL]
kwargs['version'] = OCRD_TOOL['version']
super(CalamariRecognize, self).__init__(*args, **kwargs)
def _init_calamari(self):
os.environ['TF_CPP_MIN_LOG_LEVEL'] = TF_CPP_MIN_LOG_LEVEL
checkpoints = glob(self.parameter['checkpoint'])
self.predictor = MultiPredictor(checkpoints=checkpoints)
voter_params = VoterParams()
voter_params.type = VoterParams.Type.Value(
self.parameter['voter'].upper())
self.voter = voter_from_proto(voter_params)
def process(self):
"""
Performs the recognition.
"""
assert_file_grp_cardinality(self.input_file_grp, 1)
assert_file_grp_cardinality(self.output_file_grp, 1)
self._init_calamari()
for (n, input_file) in enumerate(self.input_files):
page_id = input_file.pageId or input_file.ID
log.info("INPUT FILE %i / %s", n, page_id)
pcgts = page_from_file(self.workspace.download_file(input_file))
page = pcgts.get_Page()
page_image, page_xywh, page_image_info = self.workspace.image_from_page(
page, page_id)
for region in pcgts.get_Page().get_TextRegion():
region_image, region_xywh = self.workspace.image_from_segment(
region, page_image, page_xywh)
textlines = region.get_TextLine()
log.info("About to recognize %i lines of region '%s'",
len(textlines), region.id)
for (line_no, line) in enumerate(textlines):
log.debug("Recognizing line '%s' in region '%s'", line.id,
region.id)
line_image, line_coords = self.workspace.image_from_segment(
line, region_image, region_xywh)
line_image_np = np.array(line_image, dtype=np.uint8)
raw_results = list(
self.predictor.predict_raw([line_image_np],
progress_bar=False))[0]
for i, p in enumerate(raw_results):
p.prediction.id = "fold_{}".format(i)
prediction = self.voter.vote_prediction_result(raw_results)
prediction.id = "voted"
# Build line text on our own
#
# Calamari does whitespace post-processing on prediction.sentence, while it does not do the same
# on prediction.positions. Do it on our own to have consistency.
#
# XXX Check Calamari's built-in post-processing on prediction.sentence
def _sort_chars(p):
"""Filter and sort chars of prediction p"""
chars = p.chars
chars = [
c for c in chars if c.char
] # XXX Note that omission probabilities are not normalized?!
chars = [
c for c in chars if c.probability >=
self.parameter['glyph_conf_cutoff']
]
chars = sorted(chars,
key=lambda k: k.probability,
reverse=True)
return chars
def _drop_leading_spaces(positions):
return list(
itertools.dropwhile(
lambda p: _sort_chars(p)[0].char == " ",
positions))
def _drop_trailing_spaces(positions):
return list(
reversed(_drop_leading_spaces(
reversed(positions))))
def _drop_double_spaces(positions):
def _drop_double_spaces_generator(positions):
last_was_space = False
for p in positions:
if p.chars[0].char == " ":
if not last_was_space:
yield p
last_was_space = True
else:
yield p
last_was_space = False
return list(_drop_double_spaces_generator(positions))
positions = prediction.positions
positions = _drop_leading_spaces(positions)
positions = _drop_trailing_spaces(positions)
positions = _drop_double_spaces(positions)
positions = list(positions)
line_text = ''.join(
_sort_chars(p)[0].char for p in positions)
if line_text != prediction.sentence:
log.warning(
"Our own line text is not the same as Calamari's: '%s' != '%s'",
line_text, prediction.sentence)
# Delete existing results
if line.get_TextEquiv():
log.warning("Line '%s' already contained text results",
line.id)
line.set_TextEquiv([])
if line.get_Word():
log.warning(
"Line '%s' already contained word segmentation",
line.id)
line.set_Word([])
# Save line results
line_conf = prediction.avg_char_probability
line.set_TextEquiv(
[TextEquivType(Unicode=line_text, conf=line_conf)])
# Save word results
#
# Calamari OCR does not provide word positions, so we infer word positions from a. text segmentation
# and b. the glyph positions. This is necessary because the PAGE XML format enforces a strict
# hierarchy of lines > words > glyphs.
def _words(s):
"""Split words based on spaces and include spaces as 'words'"""
spaces = None
word = ''
for c in s:
if c == ' ' and spaces is True:
word += c
elif c != ' ' and spaces is False:
word += c
else:
if word:
yield word
word = c
spaces = (c == ' ')
yield word
if self.parameter['textequiv_level'] in ['word', 'glyph']:
word_no = 0
i = 0
for word_text in _words(line_text):
word_length = len(word_text)
if not all(c == ' ' for c in word_text):
word_positions = positions[i:i + word_length]
word_start = word_positions[0].global_start
word_end = word_positions[-1].global_end
polygon = polygon_from_x0y0x1y1([
word_start, 0, word_end, line_image.height
])
points = points_from_polygon(
coordinates_for_segment(
polygon, None, line_coords))
# XXX Crop to line polygon?
word = WordType(id='%s_word%04d' %
(line.id, word_no),
Coords=CoordsType(points))
word.add_TextEquiv(
TextEquivType(Unicode=word_text))
if self.parameter[
'textequiv_level'] == 'glyph':
for glyph_no, p in enumerate(
word_positions):
glyph_start = p.global_start
glyph_end = p.global_end
polygon = polygon_from_x0y0x1y1([
glyph_start, 0, glyph_end,
line_image.height
])
points = points_from_polygon(
coordinates_for_segment(
polygon, None, line_coords))
glyph = GlyphType(
id='%s_glyph%04d' %
(word.id, glyph_no),
Coords=CoordsType(points))
# Add predictions (= TextEquivs)
char_index_start = 1 # Must start with 1, see https://ocr-d.github.io/page#multiple-textequivs
for char_index, char in enumerate(
_sort_chars(p),
start=char_index_start):
glyph.add_TextEquiv(
TextEquivType(
Unicode=char.char,
index=char_index,
conf=char.probability))
word.add_Glyph(glyph)
line.add_Word(word)
word_no += 1
i += word_length
_page_update_higher_textequiv_levels('line', pcgts)
# Add metadata about this operation and its runtime parameters:
metadata = pcgts.get_Metadata() # ensured by from_file()
metadata.add_MetadataItem(
MetadataItemType(
type_="processingStep",
name=self.ocrd_tool['steps'][0],
value=TOOL,
Labels=[
LabelsType(externalModel="ocrd-tool",
externalId="parameters",
Label=[
LabelType(type_=name,
value=self.parameter[name])
for name in self.parameter.keys()
])
]))
file_id = make_file_id(input_file, self.output_file_grp)
pcgts.set_pcGtsId(file_id)
self.workspace.add_file(ID=file_id,
file_grp=self.output_file_grp,
pageId=input_file.pageId,
mimetype=MIMETYPE_PAGE,
local_filename=os.path.join(
self.output_file_grp,
file_id + '.xml'),
content=to_xml(pcgts))
class CalamariRecognize(Processor):
def __init__(self, *args, **kwargs):
kwargs['ocrd_tool'] = OCRD_TOOL['tools'][TOOL]
kwargs['version'] = '%s (calamari %s, tensorflow %s)' % (
OCRD_TOOL['version'], calamari_version, tensorflow_version)
super(CalamariRecognize, self).__init__(*args, **kwargs)
if hasattr(self, 'output_file_grp'):
# processing context
self.setup()
def setup(self):
"""
Set up the model prior to processing.
"""
resolved = self.resolve_resource(self.parameter['checkpoint_dir'])
checkpoints = glob('%s/*.ckpt.json' % resolved)
self.predictor = MultiPredictor(checkpoints=checkpoints)
self.network_input_channels = self.predictor.predictors[
0].network.input_channels
#self.network_input_channels = self.predictor.predictors[0].network_params.channels # not used!
# binarization = self.predictor.predictors[0].model_params.data_preprocessor.binarization # not used!
# self.features = ('' if self.network_input_channels != 1 else
# 'binarized' if binarization != 'GRAY' else
# 'grayscale_normalized')
self.features = ''
voter_params = VoterParams()
voter_params.type = VoterParams.Type.Value(
self.parameter['voter'].upper())
self.voter = voter_from_proto(voter_params)
def process(self):
"""
Perform text recognition with Calamari on the workspace.
If ``texequiv_level`` is ``word`` or ``glyph``, then additionally create word / glyph level segments by
splitting at white space characters / glyph boundaries. In the case of ``glyph``, add all alternative character
hypotheses down to ``glyph_conf_cutoff`` confidence threshold.
"""
log = getLogger('processor.CalamariRecognize')
assert_file_grp_cardinality(self.input_file_grp, 1)
assert_file_grp_cardinality(self.output_file_grp, 1)
for (n, input_file) in enumerate(self.input_files):
page_id = input_file.pageId or input_file.ID
log.info("INPUT FILE %i / %s", n, page_id)
pcgts = page_from_file(self.workspace.download_file(input_file))
page = pcgts.get_Page()
page_image, page_coords, page_image_info = self.workspace.image_from_page(
page, page_id, feature_selector=self.features)
for region in page.get_AllRegions(classes=['Text']):
region_image, region_coords = self.workspace.image_from_segment(
region,
page_image,
page_coords,
feature_selector=self.features)
textlines = region.get_TextLine()
log.info("About to recognize %i lines of region '%s'",
len(textlines), region.id)
line_images_np = []
line_coordss = []
for line in textlines:
log.debug("Recognizing line '%s' in region '%s'", line.id,
region.id)
line_image, line_coords = self.workspace.image_from_segment(
line,
region_image,
region_coords,
feature_selector=self.features)
if ('binarized' not in line_coords['features']
and 'grayscale_normalized'
not in line_coords['features']
and self.network_input_channels == 1):
# We cannot use a feature selector for this since we don't
# know whether the model expects (has been trained on)
# binarized or grayscale images; but raw images are likely
# always inadequate:
log.warning(
"Using raw image for line '%s' in region '%s'",
line.id, region.id)
line_image = line_image if all(line_image.size) else [[0]]
line_image_np = np.array(line_image, dtype=np.uint8)
line_images_np.append(line_image_np)
line_coordss.append(line_coords)
raw_results_all = self.predictor.predict_raw(
line_images_np, progress_bar=False)
for line, line_coords, raw_results in zip(
textlines, line_coordss, raw_results_all):
for i, p in enumerate(raw_results):
p.prediction.id = "fold_{}".format(i)
prediction = self.voter.vote_prediction_result(raw_results)
prediction.id = "voted"
# Build line text on our own
#
# Calamari does whitespace post-processing on prediction.sentence, while it does not do the same
# on prediction.positions. Do it on our own to have consistency.
#
# XXX Check Calamari's built-in post-processing on prediction.sentence
def _sort_chars(p):
"""Filter and sort chars of prediction p"""
chars = p.chars
chars = [
c for c in chars if c.char
] # XXX Note that omission probabilities are not normalized?!
chars = [
c for c in chars if c.probability >=
self.parameter['glyph_conf_cutoff']
]
chars = sorted(chars,
key=lambda k: k.probability,
reverse=True)
return chars
def _drop_leading_spaces(positions):
return list(
itertools.dropwhile(
lambda p: _sort_chars(p)[0].char == " ",
positions))
def _drop_trailing_spaces(positions):
return list(
reversed(_drop_leading_spaces(
reversed(positions))))
def _drop_double_spaces(positions):
def _drop_double_spaces_generator(positions):
last_was_space = False
for p in positions:
if p.chars[0].char == " ":
if not last_was_space:
yield p
last_was_space = True
else:
yield p
last_was_space = False
return list(_drop_double_spaces_generator(positions))
positions = prediction.positions
positions = _drop_leading_spaces(positions)
positions = _drop_trailing_spaces(positions)
positions = _drop_double_spaces(positions)
positions = list(positions)
line_text = ''.join(
_sort_chars(p)[0].char for p in positions)
if line_text != prediction.sentence:
log.warning(
"Our own line text is not the same as Calamari's: '%s' != '%s'",
line_text, prediction.sentence)
# Delete existing results
if line.get_TextEquiv():
log.warning("Line '%s' already contained text results",
line.id)
line.set_TextEquiv([])
if line.get_Word():
log.warning(
"Line '%s' already contained word segmentation",
line.id)
line.set_Word([])
# Save line results
line_conf = prediction.avg_char_probability
line.set_TextEquiv(
[TextEquivType(Unicode=line_text, conf=line_conf)])
# Save word results
#
# Calamari OCR does not provide word positions, so we infer word positions from a. text segmentation
# and b. the glyph positions. This is necessary because the PAGE XML format enforces a strict
# hierarchy of lines > words > glyphs.
def _words(s):
"""Split words based on spaces and include spaces as 'words'"""
spaces = None
word = ''
for c in s:
if c == ' ' and spaces is True:
word += c
elif c != ' ' and spaces is False:
word += c
else:
if word:
yield word
word = c
spaces = (c == ' ')
yield word
if self.parameter['textequiv_level'] in ['word', 'glyph']:
word_no = 0
i = 0
for word_text in _words(line_text):
word_length = len(word_text)
if not all(c == ' ' for c in word_text):
word_positions = positions[i:i + word_length]
word_start = word_positions[0].global_start
word_end = word_positions[-1].global_end
polygon = polygon_from_x0y0x1y1([
word_start, 0, word_end, line_image.height
])
points = points_from_polygon(
coordinates_for_segment(
polygon, None, line_coords))
# XXX Crop to line polygon?
word = WordType(id='%s_word%04d' %
(line.id, word_no),
Coords=CoordsType(points))
word.add_TextEquiv(
TextEquivType(Unicode=word_text))
if self.parameter[
'textequiv_level'] == 'glyph':
for glyph_no, p in enumerate(
word_positions):
glyph_start = p.global_start
glyph_end = p.global_end
polygon = polygon_from_x0y0x1y1([
glyph_start, 0, glyph_end,
line_image.height
])
points = points_from_polygon(
coordinates_for_segment(
polygon, None, line_coords))
glyph = GlyphType(
id='%s_glyph%04d' %
(word.id, glyph_no),
Coords=CoordsType(points))
# Add predictions (= TextEquivs)
char_index_start = 1 # Must start with 1, see https://ocr-d.github.io/page#multiple-textequivs
for char_index, char in enumerate(
_sort_chars(p),
start=char_index_start):
glyph.add_TextEquiv(
TextEquivType(
Unicode=char.char,
index=char_index,
conf=char.probability))
word.add_Glyph(glyph)
line.add_Word(word)
word_no += 1
i += word_length
_page_update_higher_textequiv_levels('line', pcgts)
# Add metadata about this operation and its runtime parameters:
self.add_metadata(pcgts)
file_id = make_file_id(input_file, self.output_file_grp)
pcgts.set_pcGtsId(file_id)
self.workspace.add_file(ID=file_id,
file_grp=self.output_file_grp,
pageId=input_file.pageId,
mimetype=MIMETYPE_PAGE,
local_filename=os.path.join(
self.output_file_grp,
file_id + '.xml'),
content=to_xml(pcgts))
class OCRProcessor(Processor):
def __init__(self, options):
super().__init__(options)
self._options = options
self._ocr = self._options["ocr"]
if self._ocr == "FAKE":
self._model_path = None
self._models = []
self._line_height = 48
self._chunk_size = 1
else:
if not options["model"]:
raise click.BadParameter(
"Please specify a model path", param="model")
self._model_path = Path(options["model"])
models = list(self._model_path.glob("*.json"))
if not options["legacy_model"]:
models = [m for m in models if m.with_suffix(".h5").exists()]
if len(models) < 1:
raise FileNotFoundError(
"no Calamari models found at %s" % self._model_path)
self._models = models
self._line_height = None
self._chunk_size = None
self._predictor = None
self._voter = None
self._ignored = RegionsFilter(options["ignore"])
if self._ocr != "FULL":
logging.getLogger().setLevel(logging.INFO)
@property
def processor_name(self):
return __loader__.name
def _load_models(self):
if self._predictor is not None:
return
if self._ocr == "FAKE":
return
batch_size = self._options["batch_size"]
if batch_size > 0:
batch_size_kwargs = dict(batch_size=batch_size)
else:
batch_size_kwargs = dict()
self._chunk_size = batch_size
if len(self._models) == 1:
self._predictor = Predictor(
str(self._models[0]), **batch_size_kwargs)
self._predict_kwargs = batch_size_kwargs
self._voter = None
self._line_height = int(self._predictor.model_params.line_height)
else:
logging.info("using Calamari voting with %d models." % len(self._models))
self._predictor = MultiPredictor(
checkpoints=[str(p) for p in self._models],
**batch_size_kwargs)
self._predict_kwargs = dict()
self._voter = ConfidenceVoter()
self._line_height = int(self._predictor.predictors[0].model_params.line_height)
def artifacts(self):
return [
("reliable", Input(
Artifact.LINES, Artifact.TABLES,
stage=Stage.RELIABLE)),
("output", Output(Artifact.OCR)),
]
def process(self, page_path: Path, reliable, output):
self._load_models()
lines = reliable.lines.by_path
extractor = LineExtractor(
reliable.tables,
self._line_height,
self._options,
min_confidence=reliable.lines.min_confidence)
min_width = 6
min_height = 6
names = []
empty_names = []
images = []
for stem, im in extractor(lines, ignored=self._ignored):
if im.width >= min_width and im.height >= min_height:
names.append("/".join(stem))
images.append(np.array(im))
else:
empty_names.append("/".join(stem))
if self._ocr == "DRY":
logging.info("will ocr the following lines:\n%s" % "\n".join(sorted(names)))
return
chunk_size = self._chunk_size
if chunk_size <= 0:
chunk_size = len(images)
texts = []
if self._ocr == "FAKE":
for name in names:
texts.append("text for %s." % name)
else:
for i in range(0, len(images), chunk_size):
for prediction in self._predictor.predict_raw(
images[i:i + chunk_size], progress_bar=False, **self._predict_kwargs):
if self._voter is not None:
prediction = self._voter.vote_prediction_result(prediction)
texts.append(prediction.sentence)
with output.ocr() as zf:
for name, text in zip(names, texts):
zf.writestr("%s.txt" % name, text)
for name in empty_names:
zf.writestr("%s.txt" % name, "")
class CalamariRecognize(Processor):
def __init__(self, *args, **kwargs):
kwargs['ocrd_tool'] = OCRD_TOOL['tools']['ocrd-calamari-recognize']
super(CalamariRecognize, self).__init__(*args, **kwargs)
def _init_calamari(self):
os.environ['TF_CPP_MIN_LOG_LEVEL'] = TF_CPP_MIN_LOG_LEVEL
checkpoints = glob(self.parameter['checkpoint'])
self.predictor = MultiPredictor(checkpoints=checkpoints)
voter_params = VoterParams()
voter_params.type = VoterParams.Type.Value(self.parameter['voter'].upper())
self.voter = voter_from_proto(voter_params)
def _make_file_id(self, input_file, n):
file_id = input_file.ID.replace(self.input_file_grp, self.output_file_grp)
if file_id == input_file.ID:
file_id = concat_padded(self.output_file_grp, n)
return file_id
def process(self):
"""
Performs the recognition.
"""
self._init_calamari()
for (n, input_file) in enumerate(self.input_files):
page_id = input_file.pageId or input_file.ID
log.info("INPUT FILE %i / %s", n, page_id)
pcgts = page_from_file(self.workspace.download_file(input_file))
page = pcgts.get_Page()
page_image, page_xywh, page_image_info = self.workspace.image_from_page(page, page_id)
for region in pcgts.get_Page().get_TextRegion():
region_image, region_xywh = self.workspace.image_from_segment(region, page_image, page_xywh)
textlines = region.get_TextLine()
log.info("About to recognize %i lines of region '%s'", len(textlines), region.id)
for (line_no, line) in enumerate(textlines):
log.debug("Recognizing line '%s' in region '%s'", line_no, region.id)
line_image, line_xywh = self.workspace.image_from_segment(line, region_image, region_xywh)
line_image_np = np.array(line_image, dtype=np.uint8)
raw_results = list(self.predictor.predict_raw([line_image_np], progress_bar=False))[0]
for i, p in enumerate(raw_results):
p.prediction.id = "fold_{}".format(i)
prediction = self.voter.vote_prediction_result(raw_results)
prediction.id = "voted"
line_text = prediction.sentence
line_conf = prediction.avg_char_probability
line.set_TextEquiv([TextEquivType(Unicode=line_text, conf=line_conf)])
_page_update_higher_textequiv_levels('line', pcgts)
file_id = self._make_file_id(input_file, n)
self.workspace.add_file(
ID=file_id,
file_grp=self.output_file_grp,
pageId=input_file.pageId,
mimetype=MIMETYPE_PAGE,
local_filename=os.path.join(self.output_file_grp, file_id + '.xml'),
content=to_xml(pcgts))