def predict_raw(self, datas, batch_size=1, processes=1, progress_bar=True, apply_preproc=True): # preprocessing step if apply_preproc: datas = self.data_preproc.apply(datas, processes=processes, progress_bar=progress_bar) codec = self.codec if self.codec else Codec(self.model_params.codec.charset) # create backend self.backend.set_prediction_data(datas) self.backend.prepare(train=False) if progress_bar: out = tqdm(self.backend.prediction_step(batch_size), desc="Prediction", total=len(datas)) else: out = self.backend.prediction_step(batch_size) for p in out: yield PredictionResult(p, codec=codec, text_postproc=self.text_postproc)
def predict_raw(self, datas, batch_size=1, processes=1, progress_bar=True, apply_preproc=True): # preprocessing step if apply_preproc: datas = self.data_preproc.apply(datas, processes=processes, progress_bar=progress_bar) codec = self.codec if self.codec else Codec( self.model_params.codec.charset) # create backend self.backend.set_prediction_data(datas) self.backend.prepare(train=False) prediction_start_time = time.time() if progress_bar: out = list( tqdm(self.backend.prediction_step(batch_size), desc="Prediction", total=self.backend.num_prediction_steps(batch_size))) else: out = list(self.backend.prediction_step(batch_size)) prediction_results = [ PredictionResult( p, codec=codec, text_postproc=self.text_postproc, ) for p in out ] return prediction_results, time.time() - prediction_start_time
def __init__(self, checkpoint=None, text_postproc=None, data_preproc=None, codec=None, network=None, batch_size=1, processes=1, auto_update_checkpoints=True, with_gt=False, ): """ Predicting a dataset based on a trained model Parameters ---------- checkpoint : str, optional filepath of the checkpoint of the network to load, alternatively you can directly use a loaded `network` text_postproc : TextProcessor, optional text processor to be applied on the predicted sentence for the final output. If loaded from a checkpoint the text processor will be loaded from it. data_preproc : DataProcessor, optional data processor (must be the same as of the trained model) to be applied to the input image. If loaded from a checkpoint the text processor will be loaded from it. codec : Codec, optional Codec of the deep net to use for decoding. This parameter is only required if a custom codec is used, or a `network` has been provided instead of a `checkpoint` network : ModelInterface, optional DNN instance to used. Alternatively you can provide a `checkpoint` to load a network. batch_size : int, optional Batch size to use for prediction processes : int, optional The number of processes to use for prediction auto_update_checkpoints : bool, optional Update old models automatically (this will change the checkpoint files) with_gt : bool, optional The prediction will also output the ground truth if available else None """ self.network = network self.checkpoint = checkpoint self.processes = processes self.auto_update_checkpoints = auto_update_checkpoints self.with_gt = with_gt if checkpoint: if network: raise Exception("Either a checkpoint or a network can be provided") ckpt = Checkpoint(checkpoint, auto_update=self.auto_update_checkpoints) self.checkpoint = ckpt.ckpt_path checkpoint_params = ckpt.checkpoint self.model_params = checkpoint_params.model self.codec = codec if codec else Codec(self.model_params.codec.charset) self.network_params = self.model_params.network backend = create_backend_from_proto(self.network_params, restore=self.checkpoint, processes=processes) self.text_postproc = text_postproc if text_postproc else text_processor_from_proto(self.model_params.text_postprocessor, "post") self.data_preproc = data_preproc if data_preproc else data_processor_from_proto(self.model_params.data_preprocessor) self.network = backend.create_net( dataset=None, codec=self.codec, restore=self.checkpoint, weights=None, graph_type="predict", batch_size=batch_size) elif network: self.codec = codec self.model_params = None self.network_params = network.network_proto self.text_postproc = text_postproc self.data_preproc = data_preproc if not codec: raise Exception("A codec is required if preloaded network is used.") else: raise Exception("Either a checkpoint or a existing backend must be provided") self.out_to_in_trans = OutputToInputTransformer(self.data_preproc, self.network)
def train(self, progress_bar=False): checkpoint_params = self.checkpoint_params train_start_time = time.time() + self.checkpoint_params.total_time self.dataset.load_samples(processes=1, progress_bar=progress_bar) datas, txts = self.dataset.train_samples(skip_empty=checkpoint_params.skip_invalid_gt) if len(datas) == 0: raise Exception("Empty dataset is not allowed. Check if the data is at the correct location") if self.validation_dataset: self.validation_dataset.load_samples(processes=1, progress_bar=progress_bar) validation_datas, validation_txts = self.validation_dataset.train_samples(skip_empty=checkpoint_params.skip_invalid_gt) if len(validation_datas) == 0: raise Exception("Validation dataset is empty. Provide valid validation data for early stopping.") else: validation_datas, validation_txts = [], [] # preprocessing steps texts = self.txt_preproc.apply(txts, processes=checkpoint_params.processes, progress_bar=progress_bar) datas = self.data_preproc.apply(datas, processes=checkpoint_params.processes, progress_bar=progress_bar) validation_txts = self.txt_preproc.apply(validation_txts, processes=checkpoint_params.processes, progress_bar=progress_bar) validation_datas = self.data_preproc.apply(validation_datas, processes=checkpoint_params.processes, progress_bar=progress_bar) # compute the codec codec = self.codec if self.codec else Codec.from_texts(texts, whitelist=self.codec_whitelist) # data augmentation on preprocessed data if self.data_augmenter: datas, texts = self.data_augmenter.augment_datas(datas, texts, n_augmentations=self.n_augmentations, processes=checkpoint_params.processes, progress_bar=progress_bar) # TODO: validation data augmentation # validation_datas, validation_txts = self.data_augmenter.augment_datas(validation_datas, validation_txts, n_augmentations=0, # processes=checkpoint_params.processes, progress_bar=progress_bar) # create backend network_params = checkpoint_params.model.network network_params.features = checkpoint_params.model.line_height network_params.classes = len(codec) if self.weights: # if we load the weights, take care of codec changes as-well with open(self.weights + '.json', 'r') as f: restore_checkpoint_params = json_format.Parse(f.read(), CheckpointParams()) restore_model_params = restore_checkpoint_params.model # checks if checkpoint_params.model.line_height != network_params.features: raise Exception("The model to restore has a line height of {} but a line height of {} is requested".format( network_params.features, checkpoint_params.model.line_height )) # create codec of the same type restore_codec = codec.__class__(restore_model_params.codec.charset) # the codec changes as tuple (deletions/insertions), and the new codec is the changed old one codec_changes = restore_codec.align(codec) codec = restore_codec print("Codec changes: {} deletions, {} appends".format(len(codec_changes[0]), len(codec_changes[1]))) # The actual weight/bias matrix will be changed after loading the old weights else: codec_changes = None # store the new codec checkpoint_params.model.codec.charset[:] = codec.charset print("CODEC: {}".format(codec.charset)) # compute the labels with (new/current) codec labels = [codec.encode(txt) for txt in texts] backend = create_backend_from_proto(network_params, weights=self.weights, ) backend.set_train_data(datas, labels) backend.set_prediction_data(validation_datas) if codec_changes: backend.realign_model_labels(*codec_changes) backend.prepare(train=True) loss_stats = RunningStatistics(checkpoint_params.stats_size, checkpoint_params.loss_stats) ler_stats = RunningStatistics(checkpoint_params.stats_size, checkpoint_params.ler_stats) dt_stats = RunningStatistics(checkpoint_params.stats_size, checkpoint_params.dt_stats) early_stopping_enabled = self.validation_dataset is not None \ and checkpoint_params.early_stopping_frequency > 0 \ and checkpoint_params.early_stopping_nbest > 1 early_stopping_best_accuracy = checkpoint_params.early_stopping_best_accuracy early_stopping_best_cur_nbest = checkpoint_params.early_stopping_best_cur_nbest early_stopping_best_at_iter = checkpoint_params.early_stopping_best_at_iter early_stopping_predictor = Predictor(codec=codec, text_postproc=self.txt_postproc, backend=backend) # Start the actual training # ==================================================================================== iter = checkpoint_params.iter # helper function to write a checkpoint def make_checkpoint(base_dir, prefix, version=None): if version: checkpoint_path = os.path.abspath(os.path.join(base_dir, "{}{}.ckpt".format(prefix, version))) else: checkpoint_path = os.path.abspath(os.path.join(base_dir, "{}{:08d}.ckpt".format(prefix, iter + 1))) print("Storing checkpoint to '{}'".format(checkpoint_path)) backend.save_checkpoint(checkpoint_path) checkpoint_params.iter = iter checkpoint_params.loss_stats[:] = loss_stats.values checkpoint_params.ler_stats[:] = ler_stats.values checkpoint_params.dt_stats[:] = dt_stats.values checkpoint_params.total_time = time.time() - train_start_time checkpoint_params.early_stopping_best_accuracy = early_stopping_best_accuracy checkpoint_params.early_stopping_best_cur_nbest = early_stopping_best_cur_nbest checkpoint_params.early_stopping_best_at_iter = early_stopping_best_at_iter with open(checkpoint_path + ".json", 'w') as f: f.write(json_format.MessageToJson(checkpoint_params)) return checkpoint_path try: last_checkpoint = None # Training loop, can be interrupted by early stopping for iter in range(iter, checkpoint_params.max_iters): checkpoint_params.iter = iter iter_start_time = time.time() result = backend.train_step(checkpoint_params.batch_size) if not np.isfinite(result['loss']): print("Error: Loss is not finite! Trying to restart from last checkpoint.") if not last_checkpoint: raise Exception("No checkpoint written yet. Training must be stopped.") else: # reload also non trainable weights, such as solver-specific variables backend.load_checkpoint_weights(last_checkpoint, restore_only_trainable=False) continue loss_stats.push(result['loss']) ler_stats.push(result['ler']) dt_stats.push(time.time() - iter_start_time) if iter % checkpoint_params.display == 0: pred_sentence = self.txt_postproc.apply("".join(codec.decode(result["decoded"][0]))) gt_sentence = self.txt_postproc.apply("".join(codec.decode(result["gt"][0]))) print("#{:08d}: loss={:.8f} ler={:.8f} dt={:.8f}s".format(iter, loss_stats.mean(), ler_stats.mean(), dt_stats.mean())) print(" PRED: '{}'".format(pred_sentence)) print(" TRUE: '{}'".format(gt_sentence)) if (iter + 1) % checkpoint_params.checkpoint_frequency == 0: last_checkpoint = make_checkpoint(checkpoint_params.output_dir, checkpoint_params.output_model_prefix) if early_stopping_enabled and (iter + 1) % checkpoint_params.early_stopping_frequency == 0: print("Checking early stopping model") out = early_stopping_predictor.predict_raw(validation_datas, batch_size=checkpoint_params.batch_size, progress_bar=progress_bar, apply_preproc=False) pred_texts = [d.sentence for d in out] result = Evaluator.evaluate(gt_data=validation_txts, pred_data=pred_texts, progress_bar=progress_bar) accuracy = 1 - result["avg_ler"] if accuracy > early_stopping_best_accuracy: early_stopping_best_accuracy = accuracy early_stopping_best_cur_nbest = 1 early_stopping_best_at_iter = iter + 1 # overwrite as best model last_checkpoint = make_checkpoint( checkpoint_params.early_stopping_best_model_output_dir, prefix="", version=checkpoint_params.early_stopping_best_model_prefix, ) print("Found better model with accuracy of {:%}".format(early_stopping_best_accuracy)) else: early_stopping_best_cur_nbest += 1 print("No better model found. Currently accuracy of {:%} at iter {} (remaining nbest = {})". format(early_stopping_best_accuracy, early_stopping_best_at_iter, checkpoint_params.early_stopping_nbest - early_stopping_best_cur_nbest)) if accuracy > 0 and early_stopping_best_cur_nbest >= checkpoint_params.early_stopping_nbest: print("Early stopping now.") break except KeyboardInterrupt as e: print("Storing interrupted checkpoint") make_checkpoint(checkpoint_params.output_dir, checkpoint_params.output_model_prefix, "interrupted") raise e print("Total time {}s for {} iterations.".format(time.time() - train_start_time, iter))
if __name__ == "__main__": from calamari_ocr.ocr import Codec this_dir = os.path.dirname(os.path.realpath(__file__)) base_path = os.path.abspath(os.path.join(this_dir, "..", "..", "test", "data", "uw3_50lines", "train")) fdr = FileDataParams( num_processes=8, images=[os.path.join(base_path, "*.png")], limit=1000, ) params = DataParams( codec=Codec("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 .,:;-?+=_()*{}[]`@#$%^&'\""), downscale_factor=4, line_height=48, pre_proc=SequentialProcessorPipelineParams( run_parallel=True, processors=default_image_processors() + default_text_pre_processors() + [ AugmentationProcessorParams( modes={PipelineMode.TRAINING}, data_aug_params=DataAugmentationAmount(amount=2), ), PrepareSampleProcessorParams(modes=INPUT_PROCESSOR), ], ), post_proc=SequentialProcessorPipelineParams(run_parallel=False),
def train(self, auto_compute_codec=False, progress_bar=False): """ Launch the training Parameters ---------- auto_compute_codec : bool Compute the codec automatically based on the provided ground truth. Else provide a codec using a whitelist (faster). progress_bar : bool Show or hide any progress bar """ checkpoint_params = self.checkpoint_params train_start_time = time.time() + self.checkpoint_params.total_time # load training dataset if self.preload_training: self.dataset.preload(processes=checkpoint_params.processes, progress_bar=progress_bar) # load validation dataset if self.validation_dataset and self.preload_validation: self.validation_dataset.preload( processes=checkpoint_params.processes, progress_bar=progress_bar) # compute the codec if self.codec: codec = self.codec else: if len(self.codec_whitelist) == 0 or auto_compute_codec: codec = Codec.from_input_dataset( [self.dataset, self.validation_dataset], whitelist=self.codec_whitelist, progress_bar=progress_bar) else: codec = Codec.from_texts([], whitelist=self.codec_whitelist) # create backend network_params = checkpoint_params.model.network network_params.features = checkpoint_params.model.line_height network_params.classes = len(codec) if self.weights: # if we load the weights, take care of codec changes as-well ckpt = Checkpoint(self.weights + '.json', auto_update=self.auto_update_checkpoints) restore_checkpoint_params = ckpt.checkpoint restore_model_params = restore_checkpoint_params.model # checks if checkpoint_params.model.line_height != network_params.features: raise Exception( "The model to restore has a line height of {} but a line height of {} is requested" .format(network_params.features, checkpoint_params.model.line_height)) # create codec of the same type restore_codec = codec.__class__(restore_model_params.codec.charset) # the codec changes as tuple (deletions/insertions), and the new codec is the changed old one codec_changes = restore_codec.align(codec) codec = restore_codec print("Codec changes: {} deletions, {} appends".format( len(codec_changes[0]), len(codec_changes[1]))) # The actual weight/bias matrix will be changed after loading the old weights if all([c == 0 for c in codec_changes]): codec_changes = None # No codec changes else: codec_changes = None # store the new codec checkpoint_params.model.codec.charset[:] = codec.charset print("CODEC: {}".format(codec.charset)) backend = create_backend_from_proto( network_params, weights=self.weights, ) train_net = backend.create_net(self.dataset, codec, restore=None, weights=self.weights, graph_type="train", batch_size=checkpoint_params.batch_size) test_net = backend.create_net(self.validation_dataset, codec, restore=None, weights=self.weights, graph_type="test", batch_size=checkpoint_params.batch_size) if codec_changes: # only required on one net, since the other shares the same variables train_net.realign_model_labels(*codec_changes) train_net.prepare() test_net.prepare() if checkpoint_params.current_stage == 0: self._run_train(train_net, test_net, codec, train_start_time, progress_bar) if checkpoint_params.data_aug_retrain_on_original and self.dataset.data_augmenter and self.dataset.data_augmentation_amount > 0: print("Starting training on original data only") if checkpoint_params.current_stage == 0: checkpoint_params.current_stage = 1 checkpoint_params.iter = 0 checkpoint_params.early_stopping_best_at_iter = 0 checkpoint_params.early_stopping_best_cur_nbest = 0 checkpoint_params.early_stopping_best_accuracy = 0 self.dataset.generate_only_non_augmented = True # this is the important line! train_net.prepare() test_net.prepare() self._run_train(train_net, test_net, codec, train_start_time, progress_bar) train_net.prepare() # reset the state test_net.prepare() # to prevent blocking of tensorflow on shutdown
def train(self, progress_bar=False): """ Launch the training Parameters ---------- progress_bar : bool Show or hide any progress bar """ checkpoint_params = self.checkpoint_params train_start_time = time.time() + self.checkpoint_params.total_time self.dataset.load_samples(processes=1, progress_bar=progress_bar) datas, txts = self.dataset.train_samples( skip_empty=checkpoint_params.skip_invalid_gt) if len(datas) == 0: raise Exception( "Empty dataset is not allowed. Check if the data is at the correct location" ) if self.validation_dataset: self.validation_dataset.load_samples(processes=1, progress_bar=progress_bar) validation_datas, validation_txts = self.validation_dataset.train_samples( skip_empty=checkpoint_params.skip_invalid_gt) if len(validation_datas) == 0: raise Exception( "Validation dataset is empty. Provide valid validation data for early stopping." ) else: validation_datas, validation_txts = [], [] # preprocessing steps texts = self.txt_preproc.apply(txts, processes=checkpoint_params.processes, progress_bar=progress_bar) datas, params = [ list(a) for a in zip( *self.data_preproc.apply(datas, processes=checkpoint_params.processes, progress_bar=progress_bar)) ] validation_txts = self.txt_preproc.apply( validation_txts, processes=checkpoint_params.processes, progress_bar=progress_bar) validation_data_params = self.data_preproc.apply( validation_datas, processes=checkpoint_params.processes, progress_bar=progress_bar) # compute the codec codec = self.codec if self.codec else Codec.from_texts( texts, whitelist=self.codec_whitelist) # store original data in case data augmentation is used with a second step original_texts = texts original_datas = datas # data augmentation on preprocessed data if self.data_augmenter: datas, texts = self.data_augmenter.augment_datas( datas, texts, n_augmentations=self.n_augmentations, processes=checkpoint_params.processes, progress_bar=progress_bar) # TODO: validation data augmentation # validation_datas, validation_txts = self.data_augmenter.augment_datas(validation_datas, validation_txts, n_augmentations=0, # processes=checkpoint_params.processes, progress_bar=progress_bar) # create backend network_params = checkpoint_params.model.network network_params.features = checkpoint_params.model.line_height network_params.classes = len(codec) if self.weights: # if we load the weights, take care of codec changes as-well ckpt = Checkpoint(self.weights + '.json', auto_update=self.auto_update_checkpoints) restore_checkpoint_params = ckpt.checkpoint restore_model_params = restore_checkpoint_params.model # checks if checkpoint_params.model.line_height != network_params.features: raise Exception( "The model to restore has a line height of {} but a line height of {} is requested" .format(network_params.features, checkpoint_params.model.line_height)) # create codec of the same type restore_codec = codec.__class__(restore_model_params.codec.charset) # the codec changes as tuple (deletions/insertions), and the new codec is the changed old one codec_changes = restore_codec.align(codec) codec = restore_codec print("Codec changes: {} deletions, {} appends".format( len(codec_changes[0]), len(codec_changes[1]))) # The actual weight/bias matrix will be changed after loading the old weights if all([c == 0 for c in codec_changes]): codec_changes = None # No codec changes else: codec_changes = None # store the new codec checkpoint_params.model.codec.charset[:] = codec.charset print("CODEC: {}".format(codec.charset)) # compute the labels with (new/current) codec labels = [codec.encode(txt) for txt in texts] backend = create_backend_from_proto( network_params, weights=self.weights, ) train_net = backend.create_net(restore=None, weights=self.weights, graph_type="train", batch_size=checkpoint_params.batch_size) test_net = backend.create_net(restore=None, weights=self.weights, graph_type="test", batch_size=checkpoint_params.batch_size) train_net.set_data(datas, labels) test_net.set_data(validation_datas, validation_txts) if codec_changes: # only required on one net, since the other shares the same variables train_net.realign_model_labels(*codec_changes) train_net.prepare() test_net.prepare() if checkpoint_params.current_stage == 0: self._run_train(train_net, test_net, codec, validation_data_params, train_start_time, progress_bar) if checkpoint_params.data_aug_retrain_on_original and self.data_augmenter and self.n_augmentations > 0: print("Starting training on original data only") if checkpoint_params.current_stage == 0: checkpoint_params.current_stage = 1 checkpoint_params.iter = 0 checkpoint_params.early_stopping_best_at_iter = 0 checkpoint_params.early_stopping_best_cur_nbest = 0 checkpoint_params.early_stopping_best_accuracy = 0 train_net.set_data(original_datas, [codec.encode(txt) for txt in original_texts]) test_net.set_data(validation_datas, validation_txts) train_net.prepare() test_net.prepare() self._run_train(train_net, test_net, codec, validation_data_params, train_start_time, progress_bar)
def train(self, callbacks=None, **kwargs): callbacks = callbacks if callbacks else [] self.setup_data() # load preloaded dataset data: Data = self._data model: ModelParams = self.scenario.params.model use_training_as_validation = model.ensemble > 0 or self.params.gen.__class__ == CalamariTrainOnlyPipelineParams # Setup train pipeline train_pipeline = self.params.gen.train_data(data) if len(train_pipeline.create_data_generator()) == 0: raise ValueError("Training dataset is empty.") # Setup validation pipeline val_pipeline = None if self.params.gen.val_gen(): if model.ensemble > 0: logger.warning( "A validation dataset can not be used when training and ensemble. " "Only a training set is required. Ignoring validation data!" ) else: val_pipeline = self.params.gen.val_data(data) if len(val_pipeline.create_data_generator()) == 0: raise ValueError( "Validation dataset is empty. Provide valid validation data for early stopping. " "Alternative select train only data generator mode.") if self.params.gen.train_data(data).generator_params.preload: # preload before codec was created (not all processors can be applied, yet) data.preload(progress_bar=self._params.progress_bar) train_pipeline = self.params.gen.train_data(data) if val_pipeline: val_pipeline = self.params.gen.val_data(data) # compute the codec codec = data.params.codec if not codec: if self._params.codec.auto_compute or len( self._params.codec.resolved_include_chars) == 0: codec = Codec.from_input_dataset( filter(lambda x: x, [train_pipeline, val_pipeline]), codec_construction_params=self._params.codec, progress_bar=self._params.progress_bar, ) else: codec = Codec(list(self._params.codec.resolved_include_chars)) data.params.codec = codec model.classes = codec.size() if self.checkpoint: # if we load the weights, take care of codec changes as-well restore_checkpoint_params = self.checkpoint.dict restore_data_params = restore_checkpoint_params["scenario"]["data"] # checks if data.params.line_height != restore_data_params["line_height"]: raise ValueError( f"The model to restore has a line height of {restore_data_params.line_height}" f" but a line height of {data.params.line_height} is requested" ) # create codec of the same type restore_codec = codec.__class__( restore_data_params["codec"]["charset"]) # the codec changes as tuple (deletions/insertions), and the new codec is the changed old one codec_changes = restore_codec.align( codec, shrink=not self._params.codec.keep_loaded) codec = restore_codec logger.info( f"Codec changes: {len(codec_changes[0])} deletions, {len(codec_changes[1])} appends" ) # The actual weight/bias matrix will be changed after loading the old weights if not any(codec_changes): codec_changes = None # No codec changes self._codec_changes = codec_changes model.classes = codec.size() data.params.codec = codec logger.info(f"CODEC: {codec.charset}") if self.params.gen.train_data(data).generator_params.preload: # preload after codec was created data.preload(progress_bar=self._params.progress_bar) train_pipeline = self.params.gen.train_data(data) if use_training_as_validation: logger.info("Using training data for validation.") assert val_pipeline is None if self._params.gen.train.preload: data._pipelines[PipelineMode.EVALUATION] = RawDataPipeline( [ s for s in train_pipeline.samples if not s.meta["augmented"] ], pipeline_params=self._params.gen.setup.train, data_base=data, generator_params=train_pipeline.generator_params, input_processors=train_pipeline._input_processors, output_processors=train_pipeline._output_processors, ).to_mode(PipelineMode.EVALUATION) else: data._pipelines[ PipelineMode.EVALUATION] = train_pipeline.to_mode( PipelineMode.EVALUATION) else: if val_pipeline is None: raise ValueError( "No validation data provided." "Set 'trainer.gen TrainOnly' to pass only training data." "Validation will be performed on the training data in this case." "Alternatively, set 'trainer.gen SplitTrain' and to use by " "default 20% of the training data for validation") last_logs = None if self._params.current_stage == 0: last_logs = super(Trainer, self).train(callbacks=callbacks, ) data_aug = self._data.params.pre_proc.processors_of_type( AugmentationProcessorParams) if (self._params.data_aug_retrain_on_original and len(data_aug) > 0 and any(p.n_augmentations != 0 for p in data_aug)): logger.info("Starting training on original data only") if self._params.current_stage == 0: self._params.current_epoch = 0 self._params.current_stage = 1 self._params.early_stopping.current = 1 # CER = 100% as initial value self._params.early_stopping.n = 0 # Remove data augmenter self._data.params.pre_proc.erase_all(AugmentationProcessorParams) # Remove augmented samples if 'preloaded" if isinstance(train_pipeline, RawDataPipeline): train_pipeline.samples = [ s for s in train_pipeline.samples if not s.meta.get("augmented", False) ] logger.info( f"Training on {len(train_pipeline.create_data_generator())} samples." ) super(Trainer, self).setup_steps_per_epoch() # replace callbacks that require steps per epoch as parameter first = True for i, cb in enumerate(self._callbacks[:]): if isinstance(cb, TensorBoardCallback): cb.steps_per_epoch = self._steps_per_epoch if isinstance(cb, TrainerCheckpointsCallback): if first: self._callbacks[ i] = self.create_train_params_logger_callback( store_params=False, store_weights=True) first = False else: self._callbacks[ i] = self.create_train_params_logger_callback( store_params=True, store_weights=False) logger_callback = next(c for c in self._callbacks if isinstance(c, LoggerCallback)) super(Trainer, self).fit() last_logs = logger_callback.last_logs logger.info("Training finished") return last_logs
def __init__(self, checkpoint=None, text_postproc=None, data_preproc=None, codec=None, network=None, batch_size=1, processes=1): """ Predicting a dataset based on a trained model Parameters ---------- checkpoint : str, optional filepath of the checkpoint of the network to load, alternatively you can directly use a loaded `network` text_postproc : TextProcessor, optional text processor to be applied on the predicted sentence for the final output. If loaded from a checkpoint the text processor will be loaded from it. data_preproc : DataProcessor, optional data processor (must be the same as of the trained model) to be applied to the input image. If loaded from a checkpoint the text processor will be loaded from it. codec : Codec, optional Codec of the deep net to use for decoding. This parameter is only required if a custom codec is used, or a `network` has been provided instead of a `checkpoint` network : ModelInterface, optional DNN instance to used. Alternatively you can provide a `checkpoint` to load a network. batch_size : int, optional Batch size to use for prediction processes : int, optional The number of processes to use for prediction """ self.network = network self.checkpoint = checkpoint self.processes = processes if checkpoint: if network: raise Exception( "Either a checkpoint or a network can be provided") with open(checkpoint + '.json', 'r') as f: checkpoint_params = json_format.Parse(f.read(), CheckpointParams()) self.model_params = checkpoint_params.model self.network_params = self.model_params.network backend = create_backend_from_proto(self.network_params, restore=self.checkpoint, processes=processes) self.network = backend.create_net(restore=self.checkpoint, weights=None, graph_type="predict", batch_size=batch_size) self.text_postproc = text_postproc if text_postproc else text_processor_from_proto( self.model_params.text_postprocessor, "post") self.data_preproc = data_preproc if data_preproc else data_processor_from_proto( self.model_params.data_preprocessor) elif network: self.model_params = None self.network_params = network.network_proto self.text_postproc = text_postproc self.data_preproc = data_preproc if not codec: raise Exception( "A codec is required if preloaded network is used.") else: raise Exception( "Either a checkpoint or a existing backend must be provided") self.codec = codec if codec else Codec(self.model_params.codec.charset) self.out_to_in_trans = OutputToInputTransformer( self.data_preproc, self.network)
def train(self, callbacks=None, **kwargs): callbacks = callbacks if callbacks else [] # load preloaded dataset data: Data = self._data model_params: ModelParams = self.scenario.params.model_params train_pipeline = data.get_train_data() if len(train_pipeline.create_data_generator()) == 0: raise ValueError("Training dataset is empty.") if data.params().val: val_pipeline = data.get_val_data() if len(val_pipeline.create_data_generator()) == 0: raise ValueError( "Validation dataset is empty. Provide valid validation data for early stopping." ) else: val_pipeline = None if self._params.preload_training: # preload before codec was created (not all processors can be applied, yet) data.preload(progress_bar=self._params.progress_bar) train_pipeline = data.get_train_data() if val_pipeline: val_pipeline = data.get_val_data() # compute the codec codec = data.params().codec if not codec: if len(self._params.codec_whitelist ) == 0 or self._params.auto_compute_codec: codec = Codec.from_input_dataset( filter(lambda x: x, [train_pipeline, val_pipeline]), whitelist=self._params.codec_whitelist, progress_bar=self._params.progress_bar) else: codec = Codec.from_texts( [], whitelist=self._params.codec_whitelist) data.params().codec = codec data.params( ).downscale_factor_ = model_params.compute_downscale_factor() model_params.classes = codec.size() if self.checkpoint: # if we load the weights, take care of codec changes as-well restore_checkpoint_params = self.checkpoint.dict restore_data_params = restore_checkpoint_params['scenario_params'][ 'data_params'] # checks if data.params( ).line_height_ != restore_data_params['line_height_']: raise ValueError( f"The model to restore has a line height of {restore_data_params.line_height_}" f" but a line height of {data.params().line_height_} is requested" ) # create codec of the same type restore_codec = codec.__class__( restore_data_params['codec']['charset']) # the codec changes as tuple (deletions/insertions), and the new codec is the changed old one codec_changes = restore_codec.align( codec, shrink=not self._params.keep_loaded_codec) codec = restore_codec logger.info( f"Codec changes: {len(codec_changes[0])} deletions, {len(codec_changes[1])} appends" ) # The actual weight/bias matrix will be changed after loading the old weights if not any(codec_changes): codec_changes = None # No codec changes else: codec_changes = None model_params.classes = codec.size() data.params().codec = codec logger.info(f"CODEC: {codec.charset}") if self._params.preload_training: # preload after codec was created data.preload(progress_bar=self._params.progress_bar) train_pipeline = data.get_train_data() if self._params.current_stage == 0: super(Trainer, self).train( callbacks=callbacks, warmstart_fn=partial(WarmstarterWithCodecAdaption, codec_changes=codec_changes), ) if self._params.data_aug_retrain_on_original and self._params.scenario_params.data_params.data_aug_params.to_abs( ) > 0: logger.info("Starting training on original data only") if self._params.current_stage == 0: self._params.current_epoch = 0 self._params.current_stage = 1 self._params.early_stopping_params.current_ = 1 # CER = 100% as initial value self._params.early_stopping_params.n_ = 0 # Remove data augmenter self._data.params().pre_processors_.sample_processors = [ p for p in self._data.params().pre_processors_.sample_processors if p.name != AugmentationProcessor.__name__ ] # Remove augmented samples if 'preloaded" if isinstance(train_pipeline, RawDataPipeline): train_pipeline.samples = [ s for s in train_pipeline.samples if not s.meta.get('augmented', False) ] logger.info( f"Training on {len(train_pipeline.create_data_generator())} samples." ) super(Trainer, self).setup_steps_per_epoch() # replace callbacks that require steps per epoch as parameter tb_callback = next(cb for cb in self._callbacks if isinstance(cb, TensorBoardCallback)) tb_callback.steps_per_epoch = self._steps_per_epoch i = next(i for i, cb in enumerate(self._callbacks) if isinstance(cb, TrainParamsLoggerCallback)) del self._callbacks[i] self._callbacks.insert(i, self.create_train_params_logger_callback()) super(Trainer, self).fit() logger.info("Training finished")
def train(self, auto_compute_codec=False, progress_bar=False, training_callback=ConsoleTrainingCallback()): """ Launch the training Parameters ---------- auto_compute_codec : bool Compute the codec automatically based on the provided ground truth. Else provide a codec using a whitelist (faster). progress_bar : bool Show or hide any progress bar training_callback : TrainingCallback Callback for the training process (e.g., for displaying the current cer, loss in the console) """ with ExitStackWithPop() as exit_stack: checkpoint_params = self.checkpoint_params train_start_time = time.time() + self.checkpoint_params.total_time exit_stack.enter_context(self.dataset) if self.validation_dataset: exit_stack.enter_context(self.validation_dataset) # load training dataset if self.preload_training: new_dataset = self.dataset.to_raw_input_dataset(processes=checkpoint_params.processes, progress_bar=progress_bar) exit_stack.pop(self.dataset) self.dataset = new_dataset exit_stack.enter_context(self.dataset) # load validation dataset if self.validation_dataset and self.preload_validation: new_dataset = self.validation_dataset.to_raw_input_dataset(processes=checkpoint_params.processes, progress_bar=progress_bar) exit_stack.pop(self.validation_dataset) self.validation_dataset = new_dataset exit_stack.enter_context(self.validation_dataset) # compute the codec if self.codec: codec = self.codec else: if len(self.codec_whitelist) == 0 or auto_compute_codec: codec = Codec.from_input_dataset([self.dataset, self.validation_dataset], whitelist=self.codec_whitelist, progress_bar=progress_bar) else: codec = Codec.from_texts([], whitelist=self.codec_whitelist) # create backend network_params = checkpoint_params.model.network network_params.features = checkpoint_params.model.line_height if self.weights: # if we load the weights, take care of codec changes as-well ckpt = Checkpoint(self.weights + '.json', auto_update=self.auto_update_checkpoints) restore_checkpoint_params = ckpt.checkpoint restore_model_params = restore_checkpoint_params.model # checks if checkpoint_params.model.line_height != network_params.features: raise Exception("The model to restore has a line height of {} but a line height of {} is requested".format( network_params.features, checkpoint_params.model.line_height )) # create codec of the same type restore_codec = codec.__class__(restore_model_params.codec.charset) # the codec changes as tuple (deletions/insertions), and the new codec is the changed old one codec_changes = restore_codec.align(codec, shrink=not self.keep_loaded_codec) codec = restore_codec print("Codec changes: {} deletions, {} appends".format(len(codec_changes[0]), len(codec_changes[1]))) # The actual weight/bias matrix will be changed after loading the old weights if all([c == 0 for c in codec_changes]): codec_changes = None # No codec changes else: codec_changes = None # store the new codec network_params.classes = len(codec) checkpoint_params.model.codec.charset[:] = codec.charset print("CODEC: {}".format(codec.charset)) backend = create_backend_from_checkpoint( checkpoint_params=checkpoint_params, processes=checkpoint_params.processes, ) train_net = backend.create_net(codec, graph_type="train", checkpoint_to_load=Checkpoint(self.weights) if self.weights else None, batch_size=checkpoint_params.batch_size, codec_changes=codec_changes) if checkpoint_params.current_stage == 0: self._run_train(train_net, train_start_time, progress_bar, self.dataset, self.validation_dataset, training_callback) if checkpoint_params.data_aug_retrain_on_original and self.data_augmenter and self.n_augmentations != 0: print("Starting training on original data only") if checkpoint_params.current_stage == 0: checkpoint_params.current_stage = 1 checkpoint_params.iter = 0 checkpoint_params.early_stopping_best_at_iter = 0 checkpoint_params.early_stopping_best_cur_nbest = 0 checkpoint_params.early_stopping_best_accuracy = 0 self.dataset.generate_only_non_augmented = True # this is the important line! self._run_train(train_net, train_start_time, progress_bar, self.dataset, self.validation_dataset, training_callback)
def train(self, auto_compute_codec=False, progress_bar=False): """ Launch the training Parameters ---------- progress_bar : bool Show or hide any progress bar """ checkpoint_params = self.checkpoint_params train_start_time = time.time() + self.checkpoint_params.total_time # load training dataset if self.preload_training: self.dataset.preload(processes=checkpoint_params.processes, progress_bar=progress_bar) # load validation dataset if self.validation_dataset and self.preload_validation: self.validation_dataset.preload(processes=checkpoint_params.processes, progress_bar=progress_bar) # compute the codec if self.codec: codec = self.codec else: if len(self.codec_whitelist) == 0 or auto_compute_codec: codec = Codec.from_input_dataset([self.dataset, self.validation_dataset], whitelist=self.codec_whitelist, progress_bar=True) else: codec = Codec.from_texts([], whitelist=self.codec_whitelist) # create backend network_params = checkpoint_params.model.network network_params.features = checkpoint_params.model.line_height network_params.classes = len(codec) if self.weights: # if we load the weights, take care of codec changes as-well ckpt = Checkpoint(self.weights + '.json', auto_update=self.auto_update_checkpoints) restore_checkpoint_params = ckpt.checkpoint restore_model_params = restore_checkpoint_params.model # checks if checkpoint_params.model.line_height != network_params.features: raise Exception("The model to restore has a line height of {} but a line height of {} is requested".format( network_params.features, checkpoint_params.model.line_height )) # create codec of the same type restore_codec = codec.__class__(restore_model_params.codec.charset) # the codec changes as tuple (deletions/insertions), and the new codec is the changed old one codec_changes = restore_codec.align(codec) codec = restore_codec print("Codec changes: {} deletions, {} appends".format(len(codec_changes[0]), len(codec_changes[1]))) # The actual weight/bias matrix will be changed after loading the old weights if all([c == 0 for c in codec_changes]): codec_changes = None # No codec changes else: codec_changes = None # store the new codec checkpoint_params.model.codec.charset[:] = codec.charset print("CODEC: {}".format(codec.charset)) backend = create_backend_from_proto(network_params, weights=self.weights, ) train_net = backend.create_net(self.dataset, codec, restore=None, weights=self.weights, graph_type="train", batch_size=checkpoint_params.batch_size) test_net = backend.create_net(self.validation_dataset, codec, restore=None, weights=self.weights, graph_type="test", batch_size=checkpoint_params.batch_size) if codec_changes: # only required on one net, since the other shares the same variables train_net.realign_model_labels(*codec_changes) train_net.prepare() test_net.prepare() if checkpoint_params.current_stage == 0: self._run_train(train_net, test_net, codec, train_start_time, progress_bar) if checkpoint_params.data_aug_retrain_on_original and self.dataset.data_augmenter and self.dataset.data_augmentation_amount > 0: print("Starting training on original data only") if checkpoint_params.current_stage == 0: checkpoint_params.current_stage = 1 checkpoint_params.iter = 0 checkpoint_params.early_stopping_best_at_iter = 0 checkpoint_params.early_stopping_best_cur_nbest = 0 checkpoint_params.early_stopping_best_accuracy = 0 self.dataset.generate_only_non_augmented = True # this is the important line! train_net.prepare() test_net.prepare() self._run_train(train_net, test_net, codec, train_start_time, progress_bar) train_net.prepare() # reset the state test_net.prepare() # to prevent blocking of tensorflow on shutdown