def uw3_trainer_params(with_validation=False,
with_split=False,
preload=True,
debug=False):
p = CalamariTestScenario.default_trainer_params()
p.scenario.debug_graph_construction = debug
p.force_eager = debug
train = FileDataParams(
images=glob_all(
[os.path.join(this_dir, "data", "uw3_50lines", "train", "*.png")]),
preload=preload,
)
if with_split:
p.gen = CalamariSplitTrainerPipelineParams(validation_split_ratio=0.2,
train=train)
elif with_validation:
p.gen.val.images = glob_all(
[os.path.join(this_dir, "data", "uw3_50lines", "test", "*.png")])
p.gen.val.preload = preload
p.gen.train = train
p.gen.__post_init__()
else:
p.gen = CalamariTrainOnlyPipelineParams(train=train)
p.gen.setup.val.batch_size = 1
p.gen.setup.val.num_processes = 1
p.gen.setup.train.batch_size = 1
p.gen.setup.train.num_processes = 1
post_init(p)
return p
def update_model(params: dict, path: str):
logger.info(f"Updating model at {path}")
trainer_params = TrainerParams.from_dict(params)
post_init(trainer_params)
scenario_params = trainer_params.scenario
scenario = CalamariScenario(scenario_params)
scenario.setup()
input_layers = scenario.data.create_input_layers()
outputs = scenario.model.build(input_layers)
pred_model = keras.models.Model(inputs=input_layers, outputs=outputs)
with h5py.File(path + '.h5', 'r') as f:
if 'layer_names' not in f.attrs and 'model_weights' in f:
f = f['model_weights']
graph: Graph = pred_model.layers[2]
load_weights_from_hdf5_group(
f, [l for l in graph.layer_instances if len(l.weights) > 0] +
[graph.logits])
logger.info(f"Writing converted model at {path}.tmp.h5")
pred_model.save(path + '.tmp.h5', include_optimizer=False)
logger.info(f"Attempting to load converted model at {path}.tmp.h5")
keras.models.load_model(
path + '.tmp.h5',
custom_objects=CalamariScenario.model_cls().all_custom_objects())
logger.info(f"Replacing old model at {path}.h5")
os.remove(path + '.h5')
os.rename(path + '.tmp.h5', path + '.h5')
logger.info(f"New model successfully written")
keras.backend.clear_session()
def create_lav(cls, lav_params: "LAVParams",
scenario_params: TScenarioParams) -> "LAV":
post_init(lav_params)
post_init(scenario_params)
return cls.lav_cls()(
params=lav_params,
scenario_params=scenario_params,
**cls.static_lav_kwargs(scenario_params),
)
def setup_trainer_params( train_dataset, output_dir, debug=False):
p = CalamariOmmr4allEnsembleScenario.default_trainer_params()
p.force_eager = debug
p.gen.train = train_dataset
#p.scenario.model. # p.gen.setup.val = val_dataset
p.output_dir = output_dir
p.best_model_prefix = "text"
p.write_checkpoints = False ##
post_init(p)
return p
def __init__(self, params: TDP, **kwargs):
assert len(
kwargs) == 0, f"Not all kwargs processed by subclasses: {kwargs}"
post_init(params)
self._params = params
self.resources = ResourceManager(params.resource_base_path)
self.resources.register_all(params)
post_init(params)
self._pipelines: Dict[PipelineMode, DataPipeline] = {}
def setup_trainer_params(preload=True, debug=False):
p = CalamariTestEnsembleScenario.default_trainer_params()
p.force_eager = debug
p.gen.train = FileDataParams(
images=glob_all([os.path.join(this_dir, "data", "uw3_50lines", "train", "*.png")]),
preload=preload,
)
post_init(p)
return p
def __init__(self, params: PredictorParams, data: "DataBase", **kwargs):
assert len(
kwargs) == 0, f"Not all kwargs processed by subclasses: {kwargs}"
post_init(params)
self._params = params
self._params.pipeline.mode = PipelineMode.PREDICTION
if params.include_targets:
self._params.pipeline.mode = PipelineMode.EVALUATION
self.device_config = DeviceConfig(self._params.device)
self._data = data
self.benchmark_results = BenchmarkResults()
self._keras_model: Optional[keras.Model] = None
def __post_init__(self):
self.scenario.default_serve_dir = f"{self.best_model_prefix}.ckpt.h5"
self.scenario.trainer_params_filename = f"{self.best_model_prefix}.ckpt.json"
self.early_stopping.best_model_name = ""
self.gen.train_gen().n_folds = self.scenario.model.ensemble
self.gen.train_gen().channels = self.scenario.data.input_channels
if self.gen.val_gen() is not None:
self.gen.val_gen().channels = self.scenario.data.input_channels
self.gen.val_gen().n_folds = self.scenario.model.ensemble
if self.network:
self.scenario.model.layers = graph_params_from_definition_string(
self.network)
post_init(self.scenario.model)
def default_trainer_params(cls):
p = super().default_trainer_params()
p.gen.setup.val.batch_size = 1
p.gen.setup.val.num_processes = 1
p.gen.setup.train.batch_size = 1
p.gen.setup.train.num_processes = 1
p.epochs = 70
#p.samples_per_epoch = 2
p.scenario.data.pre_proc.run_parallel = False
#p.model.ensemble = 1
#p.scenario.data.pre_proc.
post_init(p)
return p
def test_concat_cnn_architecture(self):
trainer_params = uw3_trainer_params()
trainer_params.scenario.model.layers = [
Conv2DLayerParams(filters=10),
MaxPool2DLayerParams(),
DilatedBlockLayerParams(filters=10),
TransposedConv2DLayerParams(filters=10),
ConcatLayerParams(
concat_indices=[1, 4]
), # corresponds to output of first and fourth layer
Conv2DLayerParams(filters=10),
BiLSTMLayerParams(hidden_nodes=10),
]
post_init(trainer_params)
cmd_line_trainer_params = parse_args([
"--network",
"conv=10,pool=2x2,db=10:2,tconv=10,concat=1:4,conv=10,lstm=10"
])
self.assertDictEqual(trainer_params.scenario.model.to_dict(),
cmd_line_trainer_params.scenario.model.to_dict())
cmd_line_trainer_params = parse_args([
"--model.layers",
"Conv",
"Pool",
"DilatedBlock",
"TConv",
"Concat",
"Conv",
"BiLSTM",
"--model.layers.0.filters",
"10",
"--model.layers.2.filters",
"10",
"--model.layers.3.filters",
"10",
"--model.layers.4.concat_indices",
"1",
"4",
"--model.layers.5.filters",
"10",
"--model.layers.6.hidden_nodes",
"10",
])
self.assertDictEqual(trainer_params.scenario.model.to_dict(),
cmd_line_trainer_params.scenario.model.to_dict())
with tempfile.TemporaryDirectory() as d:
trainer_params.output_dir = d
main(trainer_params)
def default_trainer_params(cls):
p = super().default_trainer_params()
p.scenario.model.layers = [
Conv2DLayerParams(filters=2),
MaxPool2DLayerParams(pool_size=IntVec2D(4, 4)),
BiLSTMLayerParams(hidden_nodes=2),
DropoutLayerParams(rate=0.5),
]
p.gen.setup.val.batch_size = 1
p.gen.setup.val.num_processes = 1
p.gen.setup.train.batch_size = 1
p.gen.setup.train.num_processes = 1
p.epochs = 1
p.samples_per_epoch = 2
p.scenario.data.pre_proc.run_parallel = False
post_init(p)
return p
def get_preproc_image():
data_params = Data.default_params()
data_params.skip_invalid_gt = False
data_params.pre_proc.run_parallel = False
data_params.pre_proc.processors = data_params.pre_proc.processors[:-1]
for p in data_params.pre_proc.processors_of_type(FinalPreparationProcessorParams):
p.pad = 0
post_init(data_params)
pl = Data(data_params).create_pipeline(DataPipelineParams, None)
pl.mode = PipelineMode.PREDICTION
preproc = data_params.pre_proc.create(pl)
def pp(image):
its = InputSample(
image, None, SampleMeta("001", fold_id="01")
).to_input_target_sample()
s = preproc.apply_on_sample(its)
return s.inputs
return pp
def test_pure_cnn_architecture(self):
trainer_params = uw3_trainer_params()
trainer_params.scenario.model.layers = [
Conv2DLayerParams(filters=10),
MaxPool2DLayerParams(),
Conv2DLayerParams(filters=20,
strides=IntVec2D(2, 2),
kernel_size=IntVec2D(4, 4)),
Conv2DLayerParams(filters=30),
]
post_init(trainer_params)
cmd_line_trainer_params = parse_args(
["--network", "conv=10,pool=2x2,conv=20:4x4:2x2,conv=30"])
self.assertDictEqual(trainer_params.scenario.model.to_dict(),
cmd_line_trainer_params.scenario.model.to_dict())
cmd_line_trainer_params = parse_args([
"--model.layers",
"Conv",
"Pool",
"Conv",
"Conv",
"--model.layers.0.filters",
"10",
"--model.layers.2.filters",
"20",
"--model.layers.2.kernel_size.x",
"4",
"--model.layers.2.kernel_size.y",
"4",
"--model.layers.2.strides.x",
"2",
"--model.layers.2.strides.y",
"2",
"--model.layers.3.filters",
"30",
])
self.assertDictEqual(trainer_params.scenario.model.to_dict(),
cmd_line_trainer_params.scenario.model.to_dict())
with tempfile.TemporaryDirectory() as d:
trainer_params.output_dir = d
main(trainer_params)
def get_preproc_text(rtl=False):
data_params = Data.default_params()
data_params.skip_invalid_gt = False
data_params.pre_proc.run_parallel = False
if rtl:
for p in data_params.pre_proc.processors_of_type(BidiTextProcessorParams):
p.bidi_direction = BidiDirection.RTL
post_init(data_params)
pl = Data(data_params).create_pipeline(DataPipelineParams, None)
pl.mode = PipelineMode.TARGETS
preproc = data_params.pre_proc.create(pl)
def pp(text):
its = InputSample(
None, text, SampleMeta("001", fold_id="01")
).to_input_target_sample()
s = preproc.apply_on_sample(its)
return s.targets
return pp
def test_dilated_block_architecture(self):
trainer_params = uw3_trainer_params()
trainer_params.scenario.model.layers = [
Conv2DLayerParams(filters=10),
MaxPool2DLayerParams(strides=IntVec2D(2, 2)),
DilatedBlockLayerParams(filters=10),
DilatedBlockLayerParams(filters=10),
Conv2DLayerParams(filters=10),
]
post_init(trainer_params)
cmd_line_trainer_params = parse_args(
["--network", "conv=10,pool=2x2:2x2,db=10:2,db=10:2,conv=10"])
self.assertDictEqual(trainer_params.scenario.model.to_dict(),
cmd_line_trainer_params.scenario.model.to_dict())
cmd_line_trainer_params = parse_args([
"--model.layers",
"Conv",
"Pool",
"DilatedBlock",
"DilatedBlock",
"Conv",
"--model.layers.0.filters",
"10",
"--model.layers.1.strides",
"2",
"2",
"--model.layers.2.filters",
"10",
"--model.layers.3.filters",
"10",
"--model.layers.4.filters",
"10",
])
self.assertDictEqual(trainer_params.scenario.model.to_dict(),
cmd_line_trainer_params.scenario.model.to_dict())
with tempfile.TemporaryDirectory() as d:
trainer_params.output_dir = d
main(trainer_params)
def test_pure_lstm_architecture(self):
trainer_params = uw3_trainer_params()
trainer_params.scenario.model.layers = [
BiLSTMLayerParams(hidden_nodes=10),
BiLSTMLayerParams(hidden_nodes=20),
]
post_init(trainer_params)
cmd_line_trainer_params = parse_args(["--network", "lstm=10,lstm=20"])
self.assertDictEqual(trainer_params.scenario.model.to_dict(),
cmd_line_trainer_params.scenario.model.to_dict())
cmd_line_trainer_params = parse_args([
"--model.layers",
"BiLSTM",
"BiLSTM",
"--model.layers.0.hidden_nodes",
"10",
"--model.layers.1.hidden_nodes",
"20",
])
self.assertDictEqual(trainer_params.scenario.model.to_dict(),
cmd_line_trainer_params.scenario.model.to_dict())
with tempfile.TemporaryDirectory() as d:
trainer_params.output_dir = d
main(trainer_params)
def create_predictor(cls, model: str,
params: "PredictorParams") -> "Predictor":
post_init(params)
scenario_params = cls.params_from_path(model)
data_params = scenario_params.data
post_init(data_params)
predictor = cls.predictor_cls()(
params,
cls.data_cls()(data_params,
**cls.static_data_kwargs(scenario_params)),
**cls.static_predictor_kwargs(scenario_params),
)
model_cls = cls.model_cls()
run_eagerly = params.run_eagerly
if isinstance(model, str):
model = keras.models.load_model(
os.path.join(model, "serve"),
compile=False,
custom_objects=model_cls.all_custom_objects()
if run_eagerly else model_cls.base_custom_objects(),
)
predictor.set_model(model)
return predictor
def create_multi_lav(
cls,
lav_params: "LAVParams",
scenario_params: TScenarioParams,
predictor_params: Optional["PredictorParams"] = None,
):
post_init(lav_params)
post_init(scenario_params)
post_init(predictor_params)
return MultiLAV(
lav_params,
scenario_params,
predictor_params=predictor_params
or cls.multi_predictor_cls().params_cls()(),
**cls.static_lav_kwargs(scenario_params),
)
def __init__(self, params: TP, **kwargs):
assert len(
kwargs) == 0, f"Not all kwargs processed by subclasses: {kwargs}"
post_init(params)
self.params = params
def create_trainer(cls,
trainer_params: "TrainerParams",
restore=False) -> "Trainer":
post_init(trainer_params)
return cls.trainer_cls()(trainer_params, cls(trainer_params.scenario),
restore)
def migrate3to5(trainer_params: dict) -> dict:
convert_processor_name = {
"CenterNormalizer":
"calamari_ocr.ocr.dataset.imageprocessors.center_normalizer:CenterNormalizerProcessorParams",
"DataRangeNormalizer":
"calamari_ocr.ocr.dataset.imageprocessors.data_range_normalizer:DataRangeProcessorParams",
"FinalPreparation":
"calamari_ocr.ocr.dataset.imageprocessors.final_preparation:FinalPreparationProcessorParams",
"AugmentationProcessor":
"calamari_ocr.ocr.dataset.imageprocessors.augmentation:AugmentationProcessorParams",
"StripTextProcessor":
"calamari_ocr.ocr.dataset.textprocessors.basic_text_processors:StripTextProcessorParams",
"BidiTextProcessor":
"calamari_ocr.ocr.dataset.textprocessors.basic_text_processors:BidiTextProcessorParams",
"TextNormalizer":
"calamari_ocr.ocr.dataset.textprocessors.text_normalizer:TextNormalizerProcessorParams",
"TextRegularizer":
"calamari_ocr.ocr.dataset.textprocessors.text_regularizer:TextRegularizerProcessorParams",
"PrepareSampleProcessor":
"calamari_ocr.ocr.dataset.imageprocessors.preparesample:PrepareSampleProcessorParams",
"ReshapeOutputsProcessor":
"calamari_ocr.ocr.dataset.postprocessors.reshape:ReshapeOutputsProcessorParams",
"CTCDecoderProcessor":
"calamari_ocr.ocr.dataset.postprocessors.ctcdecoder:CTCDecoderProcessorParams",
}
for name in ["scenario"]:
rename(trainer_params, name + "_params", name)
for name in ["model", "data"]:
rename(trainer_params["scenario"], name + "_params", name)
scenario = trainer_params["scenario"]
scenario["data"]["__cls__"] = "calamari_ocr.ocr.dataset.params:DataParams"
scenario["model"]["__cls__"] = "calamari_ocr.ocr.model.params:ModelParams"
data = scenario["data"]
rename(data, "line_height_", "line_height")
rename(data, "skip_invalid_gt_", "skip_invalid_gt")
rename(data, "resource_base_path_", "resource_base_path")
rename(data, "pre_processors_", "pre_proc")
rename(data, "post_processors_", "post_proc")
rename(data["pre_proc"], "sample_processors", "processors")
rename(data["post_proc"], "sample_processors", "processors")
data["post_proc"][
"__cls__"] = "tfaip.data.pipeline.processor.params:SequentialProcessorPipelineParams"
data["pre_proc"][
"__cls__"] = "tfaip.data.pipeline.processor.params:SequentialProcessorPipelineParams"
for proc in data["pre_proc"]["processors"] + data["post_proc"][
"processors"]:
if "args" in proc:
args = proc["args"]
if args:
for k, v in args.items():
proc[k] = v
del proc["args"]
name = proc["name"]
del proc["name"]
proc["__cls__"] = convert_processor_name[name]
migrate_model_params(scenario["model"])
params = CalamariScenario.params_from_dict(scenario)
post_init(params)
scenario = params.to_dict()
return {"scenario": scenario}
def create_multi_predictor(
cls, paths: List[str],
params: "PredictorParams") -> "MultiModelPredictor":
post_init(params)
predictor_cls = cls.multi_predictor_cls()
return predictor_cls.from_paths(paths, params, cls)
def create_evaluator(cls, params: EvaluatorParams,
**kwargs) -> EvaluatorBase:
post_init(params)
if cls.evaluator_cls() is None:
raise NotImplementedError
return cls.evaluator_cls()(params=params, **kwargs)
def migrate3to4(trainer_params: dict) -> dict:
convert_processor_name = {
"CenterNormalizer":
"calamari_ocr.ocr.dataset.imageprocessors.center_normalizer:CenterNormalizerProcessorParams",
"DataRangeNormalizer":
"calamari_ocr.ocr.dataset.imageprocessors.data_range_normalizer:DataRangeProcessorParams",
"FinalPreparation":
"calamari_ocr.ocr.dataset.imageprocessors.final_preparation:FinalPreparationProcessorParams",
"AugmentationProcessor":
"calamari_ocr.ocr.dataset.imageprocessors.augmentation:AugmentationProcessorParams",
"StripTextProcessor":
"calamari_ocr.ocr.dataset.textprocessors.basic_text_processors:StripTextProcessorParams",
"TextNormalizer":
"calamari_ocr.ocr.dataset.textprocessors.text_normalizer:TextNormalizerProcessorParams",
"TextRegularizer":
"calamari_ocr.ocr.dataset.textprocessors.text_regularizer:TextRegularizerProcessorParams",
"PrepareSampleProcessor":
"calamari_ocr.ocr.dataset.imageprocessors.preparesample:PrepareSampleProcessorParams",
"ReshapeOutputsProcessor":
"calamari_ocr.ocr.dataset.postprocessors.reshape:ReshapeOutputsProcessorParams",
"CTCDecoderProcessor":
"calamari_ocr.ocr.dataset.postprocessors.ctcdecoder:CTCDecoderProcessorParams",
}
for name in ['scenario']:
rename(trainer_params, name + '_params', name)
for name in ['model', 'data']:
rename(trainer_params['scenario'], name + '_params', name)
scenario = trainer_params['scenario']
scenario['data']['__cls__'] = 'calamari_ocr.ocr.dataset.params:DataParams'
scenario['model']["__cls__"] = "calamari_ocr.ocr.model.params:ModelParams"
data = scenario['data']
rename(data, 'line_height_', 'line_height')
rename(data, 'skip_invalid_gt_', 'skip_invalid_gt')
rename(data, 'resource_base_path_', 'resource_base_path')
rename(data, 'pre_processors_', 'pre_proc')
rename(data, 'post_processors_', 'post_proc')
rename(data['pre_proc'], 'sample_processors', 'processors')
rename(data['post_proc'], 'sample_processors', 'processors')
data['post_proc'][
'__cls__'] = "tfaip.data.pipeline.processor.params:SequentialProcessorPipelineParams"
data['pre_proc'][
'__cls__'] = "tfaip.data.pipeline.processor.params:SequentialProcessorPipelineParams"
for proc in data['pre_proc']['processors'] + data['post_proc'][
'processors']:
if 'args' in proc:
args = proc['args']
if args:
for k, v in args.items():
proc[k] = v
del proc['args']
name = proc['name']
del proc['name']
proc['__cls__'] = convert_processor_name[name]
migrate_model_params(scenario['model'])
params = CalamariScenario.params_from_dict(scenario)
post_init(params)
scenario = params.to_dict()
return {'scenario': scenario}
def run(self):
# temporary dir
temporary_dir = self.params.temporary_dir
if temporary_dir is None:
temporary_dir = tempfile.mkdtemp(prefix="calamari")
else:
temporary_dir = os.path.abspath(temporary_dir)
if not os.path.exists(temporary_dir):
os.makedirs(temporary_dir)
# Compute the files in the cross fold (create a CrossFold)
fold_file = os.path.join(temporary_dir, "folds.json")
cross_fold = CrossFold(
n_folds=self.params.n_folds,
data_generator_params=self.params.trainer.gen.train,
output_dir=temporary_dir,
progress_bar=self.params.trainer.progress_bar)
cross_fold.write_folds_to_json(fold_file)
# Create the json argument file for each individual training
run_args = []
seed = self.params.trainer.random_seed or -1
folds_to_run = self.params.single_fold if len(
self.params.single_fold) > 0 else range(len(cross_fold.folds))
for fold in folds_to_run:
train_files = cross_fold.train_files(fold)
test_files = cross_fold.test_files(fold)
path = os.path.join(temporary_dir, "fold_{}.json".format(fold))
with open(path, 'w') as f:
trainer_params = deepcopy(self.params.trainer)
trainer_params.gen = CalamariDefaultTrainerPipelineParams(
train=trainer_params.gen.train,
val=deepcopy(trainer_params.gen.train),
setup=trainer_params.gen.setup,
)
if cross_fold.is_h5_dataset:
tp = trainer_params.gen.train.to_dict()
del tp['__cls__']
tp["files"] = train_files
trainer_params.gen.train = Hdf5.from_dict(tp)
vp = trainer_params.gen.val.to_dict()
del vp['__cls__']
vp['files'] = test_files
trainer_params.gen.val = Hdf5.from_dict(vp)
else:
trainer_params.gen.train.images = train_files
trainer_params.gen.val.images = test_files
trainer_params.gen.val.gt_extension = trainer_params.gen.train.gt_extension
trainer_params.scenario.id = fold
trainer_params.progress_bar_mode = 2
trainer_params.output_dir = os.path.join(
temporary_dir, "fold_{}".format(fold))
trainer_params.early_stopping.best_model_output_dir = self.params.best_models_dir
trainer_params.early_stopping.best_model_name = ''
best_model_prefix = self.params.best_model_label.format(
id=fold)
trainer_params.best_model_prefix = best_model_prefix
if self.params.visible_gpus:
assert trainer_params.device.gpus is None, "Using visible_gpus with device.gpus is not supported"
trainer_params.device.gpus = [
self.params.visible_gpus[fold %
len(self.params.visible_gpus)]
]
if seed >= 0:
trainer_params.random_seed = seed + fold
if len(self.params.weights) == 1:
trainer_params.warmstart.model = self.params.weights[0]
elif len(self.params.weights) > 1:
trainer_params.warmstart.model = self.params.weights[fold]
# start from scratch via None
if trainer_params.warmstart.model:
if len(
trainer_params.warmstart.model.strip()
) == 0 or trainer_params.warmstart.model.upper() == "NONE":
trainer_params.warmstart.model = None
else:
# access model once to upgrade the model if necessary
# (can not be performed in parallel if multiple folds use the same model)
SavedCalamariModel(trainer_params.warmstart.model)
post_init(trainer_params)
json.dump(
trainer_params.to_dict(),
f,
indent=4,
)
run_args.append({
"json": path,
"args": trainer_params,
"id": fold,
'train_script': self.train_script_path,
'run': self.params.run,
'verbose': True
})
# Launch the individual processes for each training
with multiprocessing.pool.ThreadPool(
processes=self.params.max_parallel_models) as pool:
# workaround to forward keyboard interrupt
pool.map_async(train_individual_model, run_args).get()
if not self.params.keep_temporary_files:
import shutil
shutil.rmtree(temporary_dir)
