def load_test_data():
test_datagenerator = td.LoopingDataGenerator(
r.get_data_paths_base_0(),
get_filelist_within_folder_blacklisted,
dl.get_sensor_bool_dryspot,
batch_size=512,
num_validation_samples=131072,
num_test_samples=1048576,
split_load_path=r.datasets_dryspots,
split_save_path=r.save_path,
num_workers=75,
cache_path=r.cache_path,
cache_mode=td.CachingMode.Both,
dont_care_num_samples=False,
test_mode=True)
test_data = []
test_labels = []
test_set = test_datagenerator.get_test_samples()
for data, labels, _ in test_set:
test_data.extend(data)
test_labels.extend(labels)
test_data = np.array(test_data)
test_labels = np.array(test_labels)
test_labels = np.ravel(test_labels)
print("Loaded Test data.")
return test_data, test_labels
import pickle
from pathlib import Path
import torch
import Resources.training as r
from Pipeline import torch_datagenerator as td
from Pipeline.data_gather import get_filelist_within_folder_blacklisted
from Pipeline.data_loader_dryspot import DataloaderDryspots
if __name__ == "__main__":
dlds = DataloaderDryspots(divide_by_100k=False)
batch_size = 131072
generator = td.LoopingDataGenerator(r.get_data_paths_base_0(),
get_filelist_within_folder_blacklisted,
dlds.get_sensor_bool_dryspot,
num_validation_samples=131072,
num_test_samples=1048576,
batch_size=batch_size,
split_load_path=r.dataset_split,
split_save_path=Path(),
num_workers=75,
looping_strategy=None)
all_sensor_inputs = []
for i, (inputs, _, _) in enumerate(generator):
all_sensor_inputs.append(inputs)
print(i)
all_sensor_values = torch.cat(all_sensor_inputs, dim=0)
_std = all_sensor_values.std(dim=0)
_mean = all_sensor_values.mean(dim=0)
print("Std\n", _std)
from Utils.training_utils import read_cmd_params
if __name__ == "__main__":
"""
This is the starting point for training the Deconv/Conv Part of the FlowFrontNet
with 80 sensor data to Flowfront images.
"""
args = read_cmd_params()
img_size = (112, 96)
dl = DataloaderImages(image_size=img_size, sensor_indizes=((1, 4), (1, 4)))
m = ModelTrainer(
lambda: S80DeconvModelEfficient2(demo_mode=True
if args.demo is not None else False),
data_source_paths=r.get_data_paths_base_0(),
save_path=r.save_path if args.demo is None else Path(args.demo),
load_datasets_path=r.datasets_dryspots,
cache_path=r.cache_path,
batch_size=128,
train_print_frequency=100,
epochs=1000,
num_workers=75,
num_validation_samples=131072,
num_test_samples=1048576,
data_processing_function=dl.get_sensordata_and_flowfront,
data_gather_function=get_filelist_within_folder_blacklisted,
loss_criterion=torch.nn.MSELoss(),
optimizer_function=lambda params: torch.optim.AdamW(params, lr=0.0001),
classification_evaluator_function=lambda summary_writer:
SensorToFlowfrontEvaluator(summary_writer=summary_writer),
"""
if __name__ == "__main__":
Utils.custom_mlflow.logging = False
args = read_cmd_params()
print(">>> Model: Resnext")
model = ModelWrapper(models.resnext50_32x4d(pretrained=True))
eval_on_test = False
if "swt-dgx" in socket.gethostname():
print("On DGX. - Using ResNeXt. 3 input channels. New output")
filepaths = r.get_data_paths_base_0()
save_path = r.save_path
batch_size = 1024
train_print_frequency = 100
epochs = 5
num_workers = 75
num_validation_samples = 131072
# num_test_samples = 1048576
num_test_samples = 524288
data_gather_function = get_filelist_within_folder_blacklisted
data_root = r.data_root
cache_path = r.cache_path
elif "pop-os" in socket.gethostname():
print("Running local mode.")
basepath = Path("/home/lukas/rtm/rtm_files")
filepaths = [basepath]
test_labels = np.ravel(test_labels)
print("Loaded Test data.")
return test_data, test_labels
if __name__ == "__main__":
num_samples = 150000
args = read_cmd_params()
print("Using ca. 150 000 samples.")
dl = DataloaderDryspots(sensor_indizes=((0, 1), (0, 1)))
print("Created Dataloader.")
generator = td.LoopingDataGenerator(
r.get_data_paths_base_0(),
get_filelist_within_folder_blacklisted,
dl.get_sensor_bool_dryspot,
batch_size=512,
num_validation_samples=131072,
num_test_samples=1048576,
split_load_path=r.datasets_dryspots,
split_save_path=r.save_path,
num_workers=75,
cache_path=r.cache_path,
cache_mode=td.CachingMode.Both,
dont_care_num_samples=False,
test_mode=False)
print("Created Datagenerator")
train_data = []
