def test_krakentrainer_rec_bl_load(self):
training_data = [self.xml]
evaluation_data = [self.xml]
trainer = KrakenTrainer.recognition_train_gen(format_type='xml',
load=self.model,
training_data=training_data,
evaluation_data=evaluation_data)
self.assertEqual(trainer.model.seg_type, 'baselines')
self.assertIsInstance(trainer.train_set.dataset, kraken.lib.dataset.PolygonGTDataset)
def test_krakentrainer_rec_box_load(self):
training_data = self.box_lines
evaluation_data = self.box_lines
trainer = KrakenTrainer.recognition_train_gen(format_type='path',
load=self.model,
training_data=training_data,
evaluation_data=evaluation_data)
self.assertEqual(trainer.model.seg_type, 'bbox')
self.assertIsInstance(trainer.train_set.dataset, kraken.lib.dataset.GroundTruthDataset)
def learn(transcribe_path,
validation_size=0.3,
batch_size: int = 1,
lag: int = 5,
min_delta: float = 0,
learning_rate: float = 0.001,
threads: int = 1,
augment: bool = False) -> None:
"""
Creates models out of learning from the transcribe file
:param transcribe_path: The path of the data to learn from
:param validation_size: The size of validation set
:param batch_size: Batch size to learn in every epoch
:param lag: Number of iterations without any improvement that are allowed
:param min_delta: The goal min value of accuracy
:param learning_rate: Learning rate
:param threads: Number of threads to run on
:param augment: Augment the data
:return: None
"""
if (0 == os.fork()):
exec(ketos.extract(["--output", "output_directory", transcribe_path]))
os.wait()
pngs = sorted([
"output_directory/" + f for f in os.listdir("output_directory")
if "png" in f
])
random.shuffle(pngs)
trian = pngs[:int(len(pngs) * (1 - validation_size))]
test = pngs[int(len(pngs) * (1 - validation_size)):]
def _update_progress():
print('.', end='')
def _print_eval(epoch, accuracy, **kwargs):
print(
f"epoch: {epoch}, accuracy: {accuracy}, right: {kwargs['chars'] - kwargs['error']}, errors: {kwargs['error']}"
)
hp = kraken.lib.default_specs.RECOGNITION_HYPER_PARAMS
hp["batch_size"] = batch_size
hp["lag"] = lag
hp["min_delta"] = min_delta
hp["lrate"] = learning_rate
kt = KrakenTrainer.recognition_train_gen(hyper_params=hp,
training_data=trian,
evaluation_data=test,
format_type='path',
threads=threads,
augment=augment)
kt.run(_print_eval, _update_progress)
def test_krakentrainer_rec_box_path(self):
"""
Tests recognition trainer constructor with legacy path training data.
"""
training_data = self.box_lines
evaluation_data = self.box_lines
trainer = KrakenTrainer.recognition_train_gen(format_type='path',
training_data=training_data,
evaluation_data=evaluation_data)
self.assertEqual(trainer.model.seg_type, 'bbox')
self.assertIsInstance(trainer.train_set.dataset, kraken.lib.dataset.GroundTruthDataset)
def test_krakentrainer_rec_bl_dict(self):
"""
Tests recognition trainer constructor with dictionary style training data.
"""
training_data = [{'image': path.join(resources, 'bw.png'), 'text': 'foo', 'baseline': [[10, 10], [300, 10]], 'boundary': [[10, 5], [300, 5], [300, 15], [10, 15]]}]
evaluation_data = [{'image': path.join(resources, 'bw.png'), 'text': 'foo', 'baseline': [[10, 10], [300, 10]], 'boundary': [[10, 5], [300, 5], [300, 15], [10, 15]]}]
trainer = KrakenTrainer.recognition_train_gen(format_type=None,
training_data=training_data,
evaluation_data=evaluation_data)
self.assertEqual(trainer.model.seg_type, 'baselines')
self.assertIsInstance(trainer.train_set.dataset, kraken.lib.dataset.PolygonGTDataset)
def test_krakentrainer_rec_bl_append(self):
training_data = [self.xml]
evaluation_data = [self.xml]
trainer = KrakenTrainer.recognition_train_gen(format_type='xml',
load=self.model,
append=1,
spec='[Cr4,4,32]',
training_data=training_data,
evaluation_data=evaluation_data)
self.assertEqual(trainer.model.seg_type, 'baselines')
self.assertIsInstance(trainer.train_set.dataset, kraken.lib.dataset.PolygonGTDataset)
self.assertTrue(trainer.model.spec.startswith('[1,48,0,1 Cr{C_0}4,2,1,4,2 Cr{C_1}4,4,32 O{O_2}'))
def test_krakentrainer_rec_box_append(self):
training_data = self.box_lines
evaluation_data = self.box_lines
trainer = KrakenTrainer.recognition_train_gen(format_type='path',
load=self.model,
append=1,
spec='[Cr4,4,32]',
training_data=training_data,
evaluation_data=evaluation_data)
self.assertEqual(trainer.model.seg_type, 'bbox')
self.assertIsInstance(trainer.train_set.dataset, kraken.lib.dataset.GroundTruthDataset)
self.assertTrue(trainer.model.spec.startswith('[1,48,0,1 Cr{C_0}4,2,1,4,2 Cr{C_1}4,4,32 O{O_2}'))
def test_krakentrainer_rec_bl_xml(self):
"""
Tests recognition trainer constructor with XML training data.
"""
training_data = [self.xml]
evaluation_data = [self.xml]
trainer = KrakenTrainer.recognition_train_gen(format_type='xml',
training_data=training_data,
evaluation_data=evaluation_data)
self.assertEqual(trainer.model.seg_type, 'baselines')
self.assertIsInstance(trainer.train_set.dataset, kraken.lib.dataset.PolygonGTDataset)
self.assertEqual(len(trainer.train_set), 44)
self.assertEqual(len(trainer.val_set), 44)
def test_krakentrainer_rec_box_path(self):
"""
Tests recognition trainer constructor with legacy path training data.
"""
training_data = self.box_lines
evaluation_data = self.box_lines
module = RecognitionModel(format_type='path',
training_data=training_data,
evaluation_data=evaluation_data)
module.setup()
self.assertEqual(module.nn.seg_type, 'bbox')
self.assertIsInstance(module.train_set.dataset,
kraken.lib.dataset.GroundTruthDataset)
trainer = KrakenTrainer(max_steps=1)
def test_krakentrainer_rec_bl_load_both(self):
training_data = [self.xml]
evaluation_data = [self.xml]
module = RecognitionModel(format_type='xml',
model=self.model,
training_data=training_data,
evaluation_data=evaluation_data,
resize='both')
module.setup()
self.assertEqual(module.nn.seg_type, 'baselines')
self.assertIsInstance(module.train_set.dataset,
kraken.lib.dataset.PolygonGTDataset)
trainer = KrakenTrainer(max_steps=1)
self.assertEqual(module.nn.named_spec[-1].split("c")[-1], '60')
def test_krakentrainer_rec_bl_xml(self):
"""
Tests recognition trainer constructor with XML training data.
"""
training_data = [self.xml]
evaluation_data = [self.xml]
module = RecognitionModel(format_type='xml',
training_data=training_data,
evaluation_data=evaluation_data)
module.setup()
self.assertEqual(module.nn.seg_type, 'baselines')
self.assertIsInstance(module.train_set.dataset,
kraken.lib.dataset.PolygonGTDataset)
self.assertEqual(len(module.train_set.dataset), 44)
self.assertEqual(len(module.val_set.dataset), 44)
trainer = KrakenTrainer(max_steps=1)
def test_krakentrainer_rec_box_load_both(self):
"""
Tests that adaptation works in both mode.
"""
training_data = self.box_lines
evaluation_data = self.box_lines
module = RecognitionModel(format_type='path',
model=self.model,
training_data=training_data,
evaluation_data=evaluation_data,
resize='both')
module.setup()
self.assertEqual(module.nn.seg_type, 'bbox')
self.assertIsInstance(module.train_set.dataset,
kraken.lib.dataset.GroundTruthDataset)
trainer = KrakenTrainer(max_steps=1)
self.assertEqual(module.nn.named_spec[-1].split("c")[-1], '16')
def test_krakentrainer_rec_bl_append(self):
training_data = [self.xml]
evaluation_data = [self.xml]
module = RecognitionModel(format_type='xml',
model=self.model,
append=1,
spec='[Cr4,4,32]',
training_data=training_data,
evaluation_data=evaluation_data)
module.setup()
self.assertEqual(module.nn.seg_type, 'baselines')
self.assertIsInstance(module.train_set.dataset,
kraken.lib.dataset.PolygonGTDataset)
self.assertTrue(
module.nn.spec.startswith(
'[1,48,0,1 Cr{C_0}4,2,1,4,2 Cr{C_1}4,4,32 O{O_2}'))
trainer = KrakenTrainer(max_steps=1)
def test_krakentrainer_rec_box_append(self):
"""
Tests that appending new layers onto a loaded model works.
"""
training_data = self.box_lines
evaluation_data = self.box_lines
module = RecognitionModel(format_type='path',
model=self.model,
append=1,
spec='[Cr4,4,32]',
training_data=training_data,
evaluation_data=evaluation_data)
module.setup()
self.assertEqual(module.nn.seg_type, 'bbox')
self.assertIsInstance(module.train_set.dataset,
kraken.lib.dataset.GroundTruthDataset)
self.assertTrue(
module.nn.spec.startswith(
'[1,48,0,1 Cr{C_0}4,2,1,4,2 Cr{C_1}4,4,32 O{O_2}'))
trainer = KrakenTrainer(max_steps=1)
def test_krakentrainer_rec_bl_dict(self):
"""
Tests recognition trainer constructor with dictionary style training data.
"""
training_data = [{
'image': resources / 'bw.png',
'text': 'foo',
'baseline': [[10, 10], [300, 10]],
'boundary': [[10, 5], [300, 5], [300, 15], [10, 15]]
}]
evaluation_data = [{
'image': resources / 'bw.png',
'text': 'foo',
'baseline': [[10, 10], [300, 10]],
'boundary': [[10, 5], [300, 5], [300, 15], [10, 15]]
}]
module = RecognitionModel(format_type=None,
training_data=training_data,
evaluation_data=evaluation_data)
module.setup()
self.assertEqual(module.nn.seg_type, 'baselines')
self.assertIsInstance(module.train_set.dataset,
kraken.lib.dataset.PolygonGTDataset)
trainer = KrakenTrainer(max_steps=1)