def _generate_epoch(self, text_only) -> Generator[InputSample, None, None]:
filenames = list(self.params.files)
if self.mode == PipelineMode.TRAINING:
shuffle(filenames)
for filename in filenames:
basename = split_all_ext(filename)[0]
with h5py.File(filename, 'r') as f:
codec = list(map(chr, f['codec']))
if text_only:
for i, (text, idx) in enumerate(
zip(f['transcripts'],
range(len(f['transcripts'])))):
text = "".join([codec[c] for c in text])
fold_id = idx % self.params.n_folds if self.params.n_folds > 0 else -1
yield InputSample(
None, text,
SampleMeta(id=f"{basename}/{i}", fold_id=fold_id))
else:
gen = zip(f['images'], f['images_dims'], f['transcripts'],
range(len(f['images'])))
if self.mode == PipelineMode.TRAINING:
gen = list(gen)
shuffle(gen)
for i, (image, shape, text, idx) in enumerate(gen):
image = np.reshape(image, shape)
text = "".join([codec[c] for c in text])
fold_id = idx % self.params.n_folds if self.params.n_folds > 0 else -1
yield InputSample(
image, text,
SampleMeta(id=f"{basename}/{i}", fold_id=fold_id))
def _load_sample(self, sample,
text_only) -> Generator[InputSample, None, None]:
if text_only:
yield InputSample(
None, sample["text"],
SampleMeta(sample["id"], fold_id=sample["fold_id"]))
yield InputSample(sample["image"], sample["text"],
SampleMeta(sample["id"], fold_id=sample["fold_id"]))
def _load_sample(self, sample, text_only):
if text_only:
yield InputSample(
None,
self._load_gt_txt(sample["text_path"]),
SampleMeta(sample["id"], fold_id=sample["fold_id"]),
)
else:
yield InputSample(
self._load_line(sample["image_path"]),
self._load_gt_txt(sample["text_path"]),
SampleMeta(sample["id"], fold_id=sample["fold_id"]),
)
def _generate_epoch(self, text_only) -> Generator[InputSample, None, None]:
fold_id = -1
for p, page in enumerate(self.book.pages):
if self.mode in INPUT_PROCESSOR:
img = self._load_image(page.imgFile)
if self.params.binary:
img = img > 0.9
else:
img = None
for l, line in enumerate(page.getLines()):
for f, fo in enumerate(line.formats):
fold_id += 1
sample_id = "{}_{}_{}_{}".format(
split_all_ext(page.xmlFile or page.imgFile)[0], p, l,
f)
text = None
if self.mode in TARGETS_PROCESSOR:
text = fo.text
if text_only:
yield InputSample(
None, text,
SampleMeta(id=sample_id, fold_id=fold_id))
else:
cut_img = None
if self.mode in INPUT_PROCESSOR:
ly, lx = img.shape
# Cut the Image
cut_img = img[line.rect.top:-ly + line.rect.bottom,
line.rect.left:-lx +
line.rect.right, ]
# add padding as required from normal files
cut_img = np.pad(
cut_img,
((3, 3), (0, 0)),
mode="constant",
constant_values=cut_img.max(),
)
yield InputSample(
cut_img, text,
SampleMeta(id=sample_id, fold_id=fold_id))
def _load_sample(self, sample, text_only):
cache = self.cacheopen()
s = cache[sample["id"]]
text = s.attrs.get("text")
if text_only:
yield InputSample(
None,
text,
SampleMeta(sample["id"], fold_id=sample["fold_id"]),
)
else:
im = np.empty(s.shape, s.dtype)
s.read_direct(im)
yield InputSample(
im.T,
text,
SampleMeta(sample["id"], fold_id=sample["fold_id"]),
)
def _load_sample(self, sample,
text_only) -> Generator[InputSample, None, None]:
loader = PageXMLDatasetLoader(
self.mode,
self.params.non_existing_as_empty,
self.params.text_index,
self.params.skip_invalid,
self.params.skip_commented,
)
image_path, xml_path, idx = sample
img = None
if self.mode in INPUT_PROCESSOR:
img = self._load_image(image_path)
for i, sample in enumerate(loader.load(image_path, xml_path)):
fold_id = (
idx +
i) % self.params.n_folds if self.params.n_folds > 0 else -1
text = sample["text"]
orientation = sample["orientation"]
if not text_only and self.mode in INPUT_PROCESSOR:
ly, lx = img.shape[:2]
# rotate by orientation angle in clockwise direction to correct present skew
angle = orientation if orientation and orientation % 360 != 0 else 0
line_img = PageXMLReader.cutout(
img,
sample["coords"],
mode=self.params.cut_mode,
angle=angle,
cval=None,
scale=lx / sample["img_width"],
)
# add padding as required from normal files
if self.params.pad:
img = np.pad(
img,
self.params.pad,
mode="constant",
constant_values=img.max(initial=0),
)
else:
line_img = None
yield InputSample(line_img, text,
SampleMeta(id=sample["id"], fold_id=fold_id))
def pp(image):
its = InputSample(
image, None, SampleMeta("001", fold_id="01")
).to_input_target_sample()
s = preproc.apply_on_sample(its)
return s.inputs
def pp(text):
its = InputSample(
None, text, SampleMeta("001", fold_id="01")
).to_input_target_sample()
s = preproc.apply_on_sample(its)
return s.targets
def _load_sample(self, sample, text_only):
image, text = self.data_queue.get()
fold_id = -1 if self.params.n_folds <= 0 else np.random.randint(
self.params.n_folds)
yield InputSample(image, text,
SampleMeta(id=sample['id'], fold_id=fold_id))