def load_test_data(): test_datagenerator = td.LoopingDataGenerator( r.get_data_paths_base_0(), get_filelist_within_folder_blacklisted, dl.get_sensor_bool_dryspot, batch_size=512, num_validation_samples=131072, num_test_samples=1048576, split_load_path=r.datasets_dryspots, split_save_path=r.save_path, num_workers=75, cache_path=r.cache_path, cache_mode=td.CachingMode.Both, dont_care_num_samples=False, test_mode=True) test_data = [] test_labels = [] test_set = test_datagenerator.get_test_samples() for data, labels, _ in test_set: test_data.extend(data) test_labels.extend(labels) test_data = np.array(test_data) test_labels = np.array(test_labels) test_labels = np.ravel(test_labels) print("Loaded Test data.") return test_data, test_labels
import pickle from pathlib import Path import torch import Resources.training as r from Pipeline import torch_datagenerator as td from Pipeline.data_gather import get_filelist_within_folder_blacklisted from Pipeline.data_loader_dryspot import DataloaderDryspots if __name__ == "__main__": dlds = DataloaderDryspots(divide_by_100k=False) batch_size = 131072 generator = td.LoopingDataGenerator(r.get_data_paths_base_0(), get_filelist_within_folder_blacklisted, dlds.get_sensor_bool_dryspot, num_validation_samples=131072, num_test_samples=1048576, batch_size=batch_size, split_load_path=r.dataset_split, split_save_path=Path(), num_workers=75, looping_strategy=None) all_sensor_inputs = [] for i, (inputs, _, _) in enumerate(generator): all_sensor_inputs.append(inputs) print(i) all_sensor_values = torch.cat(all_sensor_inputs, dim=0) _std = all_sensor_values.std(dim=0) _mean = all_sensor_values.mean(dim=0) print("Std\n", _std)
from Utils.training_utils import read_cmd_params if __name__ == "__main__": """ This is the starting point for training the Deconv/Conv Part of the FlowFrontNet with 80 sensor data to Flowfront images. """ args = read_cmd_params() img_size = (112, 96) dl = DataloaderImages(image_size=img_size, sensor_indizes=((1, 4), (1, 4))) m = ModelTrainer( lambda: S80DeconvModelEfficient2(demo_mode=True if args.demo is not None else False), data_source_paths=r.get_data_paths_base_0(), save_path=r.save_path if args.demo is None else Path(args.demo), load_datasets_path=r.datasets_dryspots, cache_path=r.cache_path, batch_size=128, train_print_frequency=100, epochs=1000, num_workers=75, num_validation_samples=131072, num_test_samples=1048576, data_processing_function=dl.get_sensordata_and_flowfront, data_gather_function=get_filelist_within_folder_blacklisted, loss_criterion=torch.nn.MSELoss(), optimizer_function=lambda params: torch.optim.AdamW(params, lr=0.0001), classification_evaluator_function=lambda summary_writer: SensorToFlowfrontEvaluator(summary_writer=summary_writer),
""" if __name__ == "__main__": Utils.custom_mlflow.logging = False args = read_cmd_params() print(">>> Model: Resnext") model = ModelWrapper(models.resnext50_32x4d(pretrained=True)) eval_on_test = False if "swt-dgx" in socket.gethostname(): print("On DGX. - Using ResNeXt. 3 input channels. New output") filepaths = r.get_data_paths_base_0() save_path = r.save_path batch_size = 1024 train_print_frequency = 100 epochs = 5 num_workers = 75 num_validation_samples = 131072 # num_test_samples = 1048576 num_test_samples = 524288 data_gather_function = get_filelist_within_folder_blacklisted data_root = r.data_root cache_path = r.cache_path elif "pop-os" in socket.gethostname(): print("Running local mode.") basepath = Path("/home/lukas/rtm/rtm_files") filepaths = [basepath]
test_labels = np.ravel(test_labels) print("Loaded Test data.") return test_data, test_labels if __name__ == "__main__": num_samples = 150000 args = read_cmd_params() print("Using ca. 150 000 samples.") dl = DataloaderDryspots(sensor_indizes=((0, 1), (0, 1))) print("Created Dataloader.") generator = td.LoopingDataGenerator( r.get_data_paths_base_0(), get_filelist_within_folder_blacklisted, dl.get_sensor_bool_dryspot, batch_size=512, num_validation_samples=131072, num_test_samples=1048576, split_load_path=r.datasets_dryspots, split_save_path=r.save_path, num_workers=75, cache_path=r.cache_path, cache_mode=td.CachingMode.Both, dont_care_num_samples=False, test_mode=False) print("Created Datagenerator") train_data = []