def init_trainer(): dlds = DataloaderDryspots() m = ModelTrainer( lambda: model, data_source_paths=filepaths, save_path=save_path, cache_path=cache_path, batch_size=batch_size, train_print_frequency=train_print_frequency, looping_strategy=NoOpLoopingStrategy(), epochs=epochs, dummy_epoch=False, num_workers=num_workers, num_validation_samples=num_validation_samples, num_test_samples=num_test_samples, data_processing_function=dlds. get_sensor_bool_dryspot_resized_matrix, data_gather_function=get_filelist_within_folder_blacklisted, data_root=data_root, loss_criterion=torch.nn.BCELoss(), optimizer_function=lambda params: torch.optim.AdamW(params, lr=1e-4), classification_evaluator_function=lambda: BinaryClassificationEvaluator(), lr_scheduler_function=lambda optim: ExponentialLR(optim, 0.5), caching_torch=False, demo_path=None, hold_samples_in_memory=False, ) return m
def create_trainer_and_start(self, out_path, epochs=1, load_test_set=False): dlds = DataloaderFlowfrontSensor(sensor_indizes=((1, 8), (1, 8))) m = ModelTrainer( lambda: S20DryspotModelFCWide(), data_source_paths=tr_resources.get_data_paths_debug(), save_path=out_path, load_datasets_path=self.torch_dataset_resources / "reference_datasets", cache_path=None, num_validation_samples=8, num_test_samples=8, num_workers=0, epochs=epochs, data_processing_function=dlds.get_flowfront_sensor_bool_dryspot, data_gather_function=dg.get_filelist_within_folder_blacklisted, loss_criterion=torch.nn.BCELoss(), optimizer_function=lambda params: torch.optim.AdamW(params, lr=1e-4), classification_evaluator_function=lambda summary_writer: BinaryClassificationEvaluator(summary_writer=summary_writer), load_test_set_in_training_mode=load_test_set, data_root=test_resources.test_src_dir, ) return m
def test_training_ok_notok(self): dls = DataLoaderSensor() model_trainer = ModelTrainer( lambda: ERFH5_PressureSequence_Model(), self.training_data_paths, self.training_save_path, epochs=self.expected_num_epochs_during_training, data_gather_function=dg.get_filelist_within_folder, data_processing_function=dls.sensorgrid_simulationsuccess, num_validation_samples=1, num_test_samples=1, loss_criterion=torch.nn.BCELoss(), classification_evaluator_function=lambda summary_writer: BinaryClassificationEvaluator(summary_writer=summary_writer), data_root=test_resources.test_src_dir, ) model_trainer.start_training() dirs = [e for e in self.training_save_path.iterdir() if e.is_dir()] with open(dirs[0] / "output.log") as f: content = f.read() epochs = re.findall("Mean Loss on Eval", content) self.assertTrue(len(epochs) > 0)
def setUp(self): self.training_save_path = resources.test_training_out_dir self.training_data_paths = [ resources.test_training_src_dir / 'dry_spot_from_ff' ] self.expected_num_epochs_during_training = 1 dlds = DataloaderDryspots(image_size=(143, 111), ignore_useless_states=False) self.dt = ModelTrainer( lambda: DrySpotModel(), data_source_paths=self.training_data_paths, save_path=self.training_save_path, batch_size=10, epochs=self.expected_num_epochs_during_training, num_validation_samples=5, num_test_samples=5, data_gather_function=dg.get_filelist_within_folder, data_processing_function=dlds.get_flowfront_bool_dryspot, loss_criterion=torch.nn.BCELoss(), classification_evaluator_function=lambda summary_writer: BinaryClassificationEvaluator(summary_writer=summary_writer, save_path=self.training_save_path, skip_images=True), data_root=resources.test_src_dir, )
def run_eval_w_binary_classificator(output_dir, modeltrainer, chkp_p: Path, ): output_dir.mkdir(exist_ok=True) modeltrainer.inference_on_test_set( output_path=output_dir, checkpoint_path=chkp_p, classification_evaluator_function=lambda: BinaryClassificationEvaluator(output_dir, skip_images=True, with_text_overlay=True, advanced_eval=True) ) with open(output_dir / "predictions_per_run.p", "wb") as f: pickle.dump(modeltrainer.classification_evaluator.origin_tracker, f)
freeze_nlayers=8), data_source_paths=r.get_data_paths_base_0(), save_path=r.save_path if args.demo is None else Path(args.demo), load_datasets_path=r.datasets_dryspots, cache_path=r.cache_path, batch_size=2048, train_print_frequency=100, epochs=1000, num_workers=75, num_validation_samples=131072, num_test_samples=1048576, data_processing_function=dlds.get_sensor_bool_dryspot, data_gather_function=get_filelist_within_folder_blacklisted, loss_criterion=torch.nn.BCELoss(), optimizer_function=lambda params: torch.optim.AdamW(params, lr=1e-4), classification_evaluator_function=lambda summary_writer: BinaryClassificationEvaluator(summary_writer=summary_writer), lr_scheduler_function=lambda optim: ExponentialLR(optim, 0.5), demo_path=args.demo) if not args.run_eval: m.start_training() else: m.inference_on_test_set( output_path=Path(args.eval), checkpoint_path=Path(args.checkpoint_path), classification_evaluator_function=lambda summary_writer: BinaryClassificationEvaluator(Path(args.eval) / "eval_on_test_set", skip_images=True, with_text_overlay=True))
import torch import Resources.training as r from Models.erfh5_pressuresequence_CRNN import ERFH5_PressureSequence_Model from Pipeline import data_loader_sensor as dls, data_gather as dg from Trainer.ModelTrainer import ModelTrainer from Trainer.evaluation import BinaryClassificationEvaluator if __name__ == "__main__": data_source_paths = [r.data_root / "2019-07-24_16-32-40_5000p"] save_path = r.save_path cache_path = r.cache_path trainer = ModelTrainer( lambda: ERFH5_PressureSequence_Model(), data_source_paths, save_path, None, epochs=2, data_gather_function=dg.get_filelist_within_folder, data_processing_function=dls.sensorgrid_simulationsuccess, loss_criterion=torch.nn.BCELoss(), classification_evaluator_function=BinaryClassificationEvaluator(), ) trainer.start_training() print("training finished")
from Trainer.ModelTrainer import ModelTrainer from Trainer.evaluation import BinaryClassificationEvaluator from Utils.eval_utils import run_eval_w_binary_classificator if __name__ == "__main__": dl = DataloaderDryspots(sensor_indizes=((1, 8), (1, 8)), aux_info=True) checkpoint_p = r.chkp_S20_densenet_baseline_full_trainingset adv_output_dir = checkpoint_p.parent / "advanced_eval" m = ModelTrainer( lambda: S20DryspotModelFCWide(), data_source_paths=r.get_data_paths_base_0(), save_path=r.save_path, load_datasets_path=r.datasets_dryspots, cache_path=r.cache_path, batch_size=32768, train_print_frequency=100, epochs=1000, num_workers=75, num_validation_samples=131072, num_test_samples=1048576, data_processing_function=dl.get_sensor_bool_dryspot, data_gather_function=get_filelist_within_folder_blacklisted, loss_criterion=torch.nn.BCELoss(), optimizer_function=lambda params: torch.optim.AdamW(params, lr=0.0001), classification_evaluator_function=lambda summary_writer: BinaryClassificationEvaluator(summary_writer=summary_writer), caching_torch=False) run_eval_w_binary_classificator(adv_output_dir, m, checkpoint_p)
from Trainer.ModelTrainer import ModelTrainer from Trainer.evaluation import BinaryClassificationEvaluator from Utils.eval_utils import run_eval_w_binary_classificator if __name__ == "__main__": dl = DataloaderDryspots(sensor_indizes=((1, 4), (1, 4)), aux_info=True) checkpoint_p = r.chkp_S80_densenet_baseline adv_output_dir = checkpoint_p.parent / "advanced_eval" m = ModelTrainer( lambda: S80DryspotModelFCWide(), data_source_paths=r.get_data_paths_base_0(), save_path=r.save_path, dataset_split_path=r.dataset_split, cache_path=r.cache_path, batch_size=2048, train_print_frequency=100, epochs=1000, num_workers=75, num_validation_samples=131072, num_test_samples=1048576, data_processing_function=dl.get_sensor_bool_dryspot, data_gather_function=get_filelist_within_folder_blacklisted, loss_criterion=torch.nn.BCELoss(), optimizer_function=lambda params: torch.optim.AdamW(params, lr=0.0001), classification_evaluator_function=lambda: BinaryClassificationEvaluator(), caching_torch=False) run_eval_w_binary_classificator(adv_output_dir, m, checkpoint_p)
looping_strategy=NoOpLoopingStrategy(), epochs=epochs, dummy_epoch=False, num_workers=num_workers, num_validation_samples=num_validation_samples, num_test_samples=num_test_samples, data_processing_function=dlds.get_sensor_bool_dryspot, data_gather_function=get_filelist_within_folder_blacklisted, data_root=data_root, loss_criterion=torch.nn.BCELoss(), optimizer_function=lambda params: torch.optim.AdamW( params, lr=1e-4), classification_evaluator_function=lambda: BinaryClassificationEvaluator(), lr_scheduler_function=lambda optim: ExponentialLR(optim, 0.5), caching_torch=False, demo_path=None, hold_samples_in_memory=False, ) return m print("Starting training.") m = init_trainer() m.start_training() print("Training finished. Starting evaluation") m.inference_on_test_set( classification_evaluator_function=lambda: BinaryClassificationEvaluator(save_path / "eval_on_test_set", skip_images=True, with_text_overlay=True) )
data_source_paths=filepaths, save_path=save_path, cache_path=cache_path, batch_size=batch_size, train_print_frequency=train_print_frequency, epochs=epochs, dummy_epoch=True, num_workers=num_workers, num_validation_samples=num_validation_samples, num_test_samples=num_test_samples, data_processing_function=dlm.get_sensor_dryspot_mesh, data_gather_function=get_filelist_within_folder_blacklisted, data_root=data_root, loss_criterion=torch.nn.BCELoss(), optimizer_function=lambda params: torch.optim.SGD(params, lr=0.001), classification_evaluator_function=lambda: BinaryClassificationEvaluator(), lr_scheduler_function=None, caching_torch=False, demo_path=None, drop_last_batch=True) print("Starting training routine.") m.start_training() print("Training finished. Starting test.") m.inference_on_test_set( classification_evaluator_function=lambda summary_writer: BinaryClassificationEvaluator(save_path / "eval_on_test_set", skip_images=True, with_text_overlay=True))
checkpoint_path=Path(checkpoint_path), freeze_nlayers=9), data_source_paths=r.get_data_paths_base_0(), save_path=r.save_path if args.demo is None else Path(args.demo), dataset_split_path=r.dataset_split, cache_path=r.cache_path, batch_size=2048, train_print_frequency=100, epochs=1000, num_workers=75, num_validation_samples=131072, num_test_samples=1048576, data_processing_function=dl.get_sensor_bool_dryspot, data_gather_function=get_filelist_within_folder_blacklisted, loss_criterion=torch.nn.MSELoss(), optimizer_function=lambda params: torch.optim.AdamW(params, lr=0.0001), classification_evaluator_function=lambda: BinaryClassificationEvaluator(), lr_scheduler_function=lambda optim: ExponentialLR(optim, 0.5), demo_path=args.demo) if not args.run_eval: m.start_training() else: m.inference_on_test_set( Path(args.eval), Path(args.checkpoint_path), lambda: BinaryClassificationEvaluator( Path(args.eval) / "eval_on_test_set"), )
round_at=0.8), data_source_paths=r.get_data_paths_base_0(), save_path=r.save_path if args.demo is None else Path(args.demo), dataset_split_path=r.dataset_split, cache_path=r.cache_path, batch_size=2048, train_print_frequency=100, epochs=1000, num_workers=75, num_validation_samples=131072, num_test_samples=1048576, data_processing_function=dl.get_sensor_bool_dryspot, data_gather_function=get_filelist_within_folder_blacklisted, loss_criterion=torch.nn.BCELoss(), optimizer_function=lambda params: torch.optim.AdamW(params, lr=1e-4), classification_evaluator_function=lambda: BinaryClassificationEvaluator(), lr_scheduler_function=lambda optim: ExponentialLR(optim, 0.5), demo_path=args.demo ) if not args.run_eval: m.start_training() else: m.inference_on_test_set( Path(args.eval), Path(args.checkpoint_path), lambda: BinaryClassificationEvaluator( Path(args.eval) / "eval_on_test_set", ), )
m = ModelTrainer(lambda: S1140DryspotModelFCWide(), data_source_paths=r.get_data_paths_debug(), save_path=r.save_path, dataset_split_path=r.dataset_split, cache_path=r.cache_path, batch_size=2048, train_print_frequency=100, epochs=100, num_workers=75, num_validation_samples=512, # 131072, num_test_samples=1024, # 1048576, data_processing_function=dlds.get_flowfront_sensor_bool_dryspot, data_gather_function=get_filelist_within_folder_blacklisted, loss_criterion=torch.nn.BCELoss(), optimizer_function=lambda params: torch.optim.AdamW(params, lr=1e-4), classification_evaluator_function=lambda: BinaryClassificationEvaluator(), dont_care_num_samples=True ) if not args.run_eval: m.start_training() else: m.inference_on_test_set( output_path=Path(args.eval), checkpoint_path=Path(args.checkpoint_path), classification_evaluator_function=lambda: BinaryClassificationEvaluator( Path(args.eval) / "eval_on_test_set", skip_images=True, with_text_overlay=True) )
from Trainer.evaluation import BinaryClassificationEvaluator from Utils.eval_utils import run_eval_w_binary_classificator if __name__ == "__main__": dl = DataloaderDryspots(sensor_indizes=((1, 8), (1, 8)), aux_info=True) checkpoint_p = r.chkp_S20_densenet_baseline_full_trainingset adv_output_dir = checkpoint_p.parent / "advanced_eval" m = ModelTrainer( lambda: S20DryspotModelFCWide(), data_source_paths=r.get_data_paths_base_0(), save_path=r.save_path, dataset_split_path=r.dataset_split, cache_path=r.cache_path, batch_size=32768, train_print_frequency=100, epochs=1000, num_workers=75, num_validation_samples=131072, num_test_samples=1048576, data_processing_function=dl.get_sensor_bool_dryspot, data_gather_function=get_filelist_within_folder_blacklisted, loss_criterion=torch.nn.BCELoss(), optimizer_function=lambda params: torch.optim.AdamW(params, lr=0.0001), classification_evaluator_function=lambda: BinaryClassificationEvaluator(), caching_torch=False ) run_eval_w_binary_classificator(adv_output_dir, m, checkpoint_p)
r.save_path, dataset_split_path=None, # r.dataset_split, data_root=data_root, cache_path=r.cache_path, batch_size=batch_size, epochs=num_epochs, num_workers=num_workers, num_validation_samples=num_val, num_test_samples=num_test, data_processing_function=dl.get_sensor_bool_dryspot_runlevel, data_gather_function=get_filelist_within_folder_blacklisted, loss_criterion=torch.nn.BCELoss(), optimizer_function=lambda params: torch.optim.AdamW(params, lr=lr), classification_evaluator_function=lambda: BinaryClassificationEvaluator(skip_images=not create_data_plots, max_epochs=num_epochs, data_loader=dl), # lr_scheduler_function=lambda optim: ExponentialLR(optim, 0.5), dummy_epoch=False, caching_torch=use_cache, run_name=run_name, save_in_mlflow_directly=True) if not args.run_eval: m.start_training() else: m.inference_on_test_set( Path(args.eval), Path(args.checkpoint_path), lambda: BinaryClassificationEvaluator(save_path=Path(args.eval) / "eval_on_test_set", skip_images=False))
m = ModelTrainer( lambda: DrySpotModel(), data_source_paths=r.get_data_paths_base_0(), save_path=r.save_path, load_datasets_path=r.datasets_dryspots, cache_path=r.cache_path, batch_size=2048, train_print_frequency=100, epochs=1000, num_workers=75, num_validation_samples=131072, num_test_samples=1048576, data_processing_function=dl.get_flowfront_bool_dryspot, data_gather_function=get_filelist_within_folder_blacklisted, loss_criterion=torch.nn.MSELoss(), optimizer_function=lambda params: torch.optim.AdamW(params, lr=0.0001), classification_evaluator_function=lambda summary_writer: BinaryClassificationEvaluator(summary_writer=summary_writer), ) if not args.run_eval: m.start_training() else: m.inference_on_test_set( Path(args.eval), Path(args.checkpoint_path), lambda summary_writer: BinaryClassificationEvaluator( Path(args.eval) / "eval_on_test_set", ), )