def setUp(self): self.training_save_path = resources.test_training_out_dir self.training_data_paths = [ resources.test_training_src_dir / 'dry_spot_from_ff' ] self.expected_num_epochs_during_training = 1 dlds = DataloaderDryspots(image_size=(143, 111), ignore_useless_states=False) self.dt = ModelTrainer( lambda: DrySpotModel(), data_source_paths=self.training_data_paths, save_path=self.training_save_path, batch_size=10, epochs=self.expected_num_epochs_during_training, num_validation_samples=5, num_test_samples=5, data_gather_function=dg.get_filelist_within_folder, data_processing_function=dlds.get_flowfront_bool_dryspot, loss_criterion=torch.nn.BCELoss(), classification_evaluator_function=lambda summary_writer: BinaryClassificationEvaluator(summary_writer=summary_writer, save_path=self.training_save_path, skip_images=True), data_root=resources.test_src_dir, )
def init_trainer(): dlds = DataloaderDryspots() m = ModelTrainer( lambda: model, data_source_paths=filepaths, save_path=save_path, cache_path=cache_path, batch_size=batch_size, train_print_frequency=train_print_frequency, looping_strategy=NoOpLoopingStrategy(), epochs=epochs, dummy_epoch=False, num_workers=num_workers, num_validation_samples=num_validation_samples, num_test_samples=num_test_samples, data_processing_function=dlds. get_sensor_bool_dryspot_resized_matrix, data_gather_function=get_filelist_within_folder_blacklisted, data_root=data_root, loss_criterion=torch.nn.BCELoss(), optimizer_function=lambda params: torch.optim.AdamW(params, lr=1e-4), classification_evaluator_function=lambda: BinaryClassificationEvaluator(), lr_scheduler_function=lambda optim: ExponentialLR(optim, 0.5), caching_torch=False, demo_path=None, hold_samples_in_memory=False, ) return m
dataset_paths = r.get_regular_sampled_data_paths( ) # r.get_all_data_paths() num_workers = 75 use_cache = False num_val = 50 num_test = 50 num_epochs = 15 lr = 1e-4 num_sensors = 1140 # {1140, 285, 80, 20} conv_lstm_sizes = [128, 32] fc_sizes = [2048, 512, 128] create_data_plots = True run_name = "new data (split path None)" dl = DataloaderDryspots(sensor_indizes=sensor_indices[str(num_sensors)], aux_info=True, image_size=(143, 111)) m = ModelTrainer( lambda: SensorToBinaryRunwiseModel(num_sensors, conv_lstm_sizes, fc_sizes), dataset_paths, r.save_path, dataset_split_path=None, # r.dataset_split, data_root=data_root, cache_path=r.cache_path, batch_size=batch_size, epochs=num_epochs, num_workers=num_workers, num_validation_samples=num_val, num_test_samples=num_test, data_processing_function=dl.get_sensor_bool_dryspot_runlevel,
import pickle from pathlib import Path import torch import Resources.training as r from Pipeline import torch_datagenerator as td from Pipeline.data_gather import get_filelist_within_folder_blacklisted from Pipeline.data_loader_dryspot import DataloaderDryspots if __name__ == "__main__": dlds = DataloaderDryspots(divide_by_100k=False) batch_size = 131072 generator = td.LoopingDataGenerator(r.get_data_paths_base_0(), get_filelist_within_folder_blacklisted, dlds.get_sensor_bool_dryspot, num_validation_samples=131072, num_test_samples=1048576, batch_size=batch_size, split_load_path=r.dataset_split, split_save_path=Path(), num_workers=75, looping_strategy=None) all_sensor_inputs = [] for i, (inputs, _, _) in enumerate(generator): all_sensor_inputs.append(inputs) print(i) all_sensor_values = torch.cat(all_sensor_inputs, dim=0) _std = all_sensor_values.std(dim=0) _mean = all_sensor_values.mean(dim=0) print("Std\n", _std)
import torch import Resources.training as r from Models.erfh5_fullyConnected import S20DryspotModelFCWide from Pipeline.data_gather import get_filelist_within_folder_blacklisted from Pipeline.data_loader_dryspot import DataloaderDryspots from Trainer.ModelTrainer import ModelTrainer from Trainer.evaluation import BinaryClassificationEvaluator from Utils.eval_utils import run_eval_w_binary_classificator if __name__ == "__main__": dl = DataloaderDryspots(sensor_indizes=((1, 8), (1, 8)), aux_info=True) checkpoint_p = r.chkp_S20_densenet_baseline_full_trainingset adv_output_dir = checkpoint_p.parent / "advanced_eval" m = ModelTrainer( lambda: S20DryspotModelFCWide(), data_source_paths=r.get_data_paths_base_0(), save_path=r.save_path, dataset_split_path=r.dataset_split, cache_path=r.cache_path, batch_size=32768, train_print_frequency=100, epochs=1000, num_workers=75, num_validation_samples=131072, num_test_samples=1048576, data_processing_function=dl.get_sensor_bool_dryspot, data_gather_function=get_filelist_within_folder_blacklisted, loss_criterion=torch.nn.BCELoss(),
from Utils.training_utils import read_cmd_params if __name__ == "__main__": """ This is the second stage for training the FlowFrontNet: the DrySpotNet with 1140 sensor data to Flowfront images. Please add the path to the pretrained weights as parameter checkpoint_path to the SensorDeconvToDryspotEfficient2 Model or use the command line option --checkpoint_path. """ args = read_cmd_params() if args.demo is not None: checkpoint_path = args.checkpoint_path else: checkpoint_path = "Use your own path to checkpoint." dlds = DataloaderDryspots() m = ModelTrainer( lambda: SensorDeconvToDryspotEfficient2(pretrained="deconv_weights", checkpoint_path=Path( checkpoint_path), freeze_nlayers=8), data_source_paths=r.get_data_paths_base_0(), save_path=r.save_path if args.demo is None else Path(args.demo), load_datasets_path=r.datasets_dryspots, cache_path=r.cache_path, batch_size=2048, train_print_frequency=100, epochs=1000, num_workers=75, num_validation_samples=131072, num_test_samples=1048576,
test_labels.extend(labels) test_data = np.array(test_data) test_labels = np.array(test_labels) test_labels = np.ravel(test_labels) print("Loaded Test data.") return test_data, test_labels if __name__ == "__main__": num_samples = 150000 args = read_cmd_params() print("Using ca. 150 000 samples.") dl = DataloaderDryspots(sensor_indizes=((0, 1), (0, 1))) print("Created Dataloader.") generator = td.LoopingDataGenerator( r.get_data_paths_base_0(), get_filelist_within_folder_blacklisted, dl.get_sensor_bool_dryspot, batch_size=512, num_validation_samples=131072, num_test_samples=1048576, split_load_path=r.datasets_dryspots, split_save_path=r.save_path, num_workers=75, cache_path=r.cache_path, cache_mode=td.CachingMode.Both, dont_care_num_samples=False,
import torch import Resources.training as r from Models.erfh5_fullyConnected import S1140DryspotModelFCWide from Pipeline.data_gather import get_filelist_within_folder_blacklisted from Pipeline.data_loader_dryspot import DataloaderDryspots from Trainer.ModelTrainer import ModelTrainer from Trainer.evaluation import BinaryClassificationEvaluator from Utils.eval_utils import run_eval_w_binary_classificator if __name__ == "__main__": dl = DataloaderDryspots(aux_info=True) checkpoint_p = r.chkp_S1140_densenet_baseline_full_trainingset adv_output_dir = checkpoint_p.parent / "advanced_eval" m = ModelTrainer( lambda: S1140DryspotModelFCWide(), data_source_paths=r.get_data_paths_base_0(), save_path=r.save_path, dataset_split_path=r.dataset_split, cache_path=r.cache_path, batch_size=32768, train_print_frequency=100, epochs=1000, num_workers=75, num_validation_samples=131072, num_test_samples=1048576, data_processing_function=dl.get_sensor_bool_dryspot, data_gather_function=get_filelist_within_folder_blacklisted, loss_criterion=torch.nn.BCELoss(), optimizer_function=lambda params: torch.optim.AdamW(params, lr=1e-4),
from pathlib import Path import torch import Resources.training as r from Models.erfh5_ConvModel import DrySpotModel from Pipeline.data_gather import get_filelist_within_folder_blacklisted from Pipeline.data_loader_dryspot import DataloaderDryspots from Trainer.ModelTrainer import ModelTrainer from Trainer.evaluation import BinaryClassificationEvaluator from Utils.training_utils import read_cmd_params if __name__ == "__main__": args = read_cmd_params() dl = DataloaderDryspots(image_size=(143, 111)) m = ModelTrainer( lambda: DrySpotModel(), data_source_paths=r.get_data_paths_base_0(), save_path=r.save_path, load_datasets_path=r.datasets_dryspots, cache_path=r.cache_path, batch_size=2048, train_print_frequency=100, epochs=1000, num_workers=75, num_validation_samples=131072, num_test_samples=1048576, data_processing_function=dl.get_flowfront_bool_dryspot, data_gather_function=get_filelist_within_folder_blacklisted,
from Utils.training_utils import read_cmd_params if __name__ == "__main__": """ This is the second stage for training the FlowFrontNet: the DrySpotNet with 80 sensor data to Flowfront images. Please add the path to the pretrained weights as parameter checkpoint_path to the SensorDeconvToDryspotEfficient2 Model or use the command line option --checkpoint_path. """ args = read_cmd_params() if args.demo is not None: checkpoint_path = args.checkpoint_path else: checkpoint_path = "Use your own path to checkpoint." dl = DataloaderDryspots(sensor_indizes=((1, 4), (1, 4))) m = ModelTrainer( lambda: S80Deconv2ToDrySpotEff(demo_mode=True if args.demo is not None else False, pretrained="deconv_weights", checkpoint_path=Path(checkpoint_path), freeze_nlayers=9), data_source_paths=r.get_data_paths_base_0(), save_path=r.save_path if args.demo is None else Path(args.demo), dataset_split_path=r.dataset_split, cache_path=r.cache_path, batch_size=2048, train_print_frequency=100, epochs=1000, num_workers=75,
import Resources.training as r from Models.erfh5_fullyConnected import S20DryspotModelFCWide from Pipeline.data_gather import get_filelist_within_folder_blacklisted from Pipeline.data_loader_dryspot import DataloaderDryspots from Trainer.ModelTrainer import ModelTrainer from Trainer.evaluation import BinaryClassificationEvaluator from Utils.training_utils import read_cmd_params if __name__ == "__main__": """ This is the starting point for training the feed foward network with 20 sensor data to binary classification. """ args = read_cmd_params() dlds = DataloaderDryspots(sensor_indizes=((1, 8), (1, 8))) m = ModelTrainer( lambda: S20DryspotModelFCWide(demo_mode=True if args.demo is not None else False), data_source_paths=r.get_data_paths_base_0(), save_path=r.save_path if args.demo is None else Path(args.demo), load_datasets_path=r.datasets_dryspots, cache_path=r.cache_path, batch_size=32768, train_print_frequency=100, epochs=100, num_workers=75, num_validation_samples=131072, num_test_samples=1048576, data_processing_function=dlds.get_sensor_bool_dryspot, data_gather_function=get_filelist_within_folder_blacklisted,