def setUp(self):
self.training_save_path = resources.test_training_out_dir
self.training_data_paths = [
resources.test_training_src_dir / 'dry_spot_from_ff'
]
self.expected_num_epochs_during_training = 1
dlds = DataloaderDryspots(image_size=(143, 111),
ignore_useless_states=False)
self.dt = ModelTrainer(
lambda: DrySpotModel(),
data_source_paths=self.training_data_paths,
save_path=self.training_save_path,
batch_size=10,
epochs=self.expected_num_epochs_during_training,
num_validation_samples=5,
num_test_samples=5,
data_gather_function=dg.get_filelist_within_folder,
data_processing_function=dlds.get_flowfront_bool_dryspot,
loss_criterion=torch.nn.BCELoss(),
classification_evaluator_function=lambda summary_writer:
BinaryClassificationEvaluator(summary_writer=summary_writer,
save_path=self.training_save_path,
skip_images=True),
data_root=resources.test_src_dir,
)
def init_trainer():
dlds = DataloaderDryspots()
m = ModelTrainer(
lambda: model,
data_source_paths=filepaths,
save_path=save_path,
cache_path=cache_path,
batch_size=batch_size,
train_print_frequency=train_print_frequency,
looping_strategy=NoOpLoopingStrategy(),
epochs=epochs,
dummy_epoch=False,
num_workers=num_workers,
num_validation_samples=num_validation_samples,
num_test_samples=num_test_samples,
data_processing_function=dlds.
get_sensor_bool_dryspot_resized_matrix,
data_gather_function=get_filelist_within_folder_blacklisted,
data_root=data_root,
loss_criterion=torch.nn.BCELoss(),
optimizer_function=lambda params: torch.optim.AdamW(params,
lr=1e-4),
classification_evaluator_function=lambda:
BinaryClassificationEvaluator(),
lr_scheduler_function=lambda optim: ExponentialLR(optim, 0.5),
caching_torch=False,
demo_path=None,
hold_samples_in_memory=False,
)
return m
dataset_paths = r.get_regular_sampled_data_paths(
) # r.get_all_data_paths()
num_workers = 75
use_cache = False
num_val = 50
num_test = 50
num_epochs = 15
lr = 1e-4
num_sensors = 1140 # {1140, 285, 80, 20}
conv_lstm_sizes = [128, 32]
fc_sizes = [2048, 512, 128]
create_data_plots = True
run_name = "new data (split path None)"
dl = DataloaderDryspots(sensor_indizes=sensor_indices[str(num_sensors)],
aux_info=True,
image_size=(143, 111))
m = ModelTrainer(
lambda: SensorToBinaryRunwiseModel(num_sensors, conv_lstm_sizes,
fc_sizes),
dataset_paths,
r.save_path,
dataset_split_path=None, # r.dataset_split,
data_root=data_root,
cache_path=r.cache_path,
batch_size=batch_size,
epochs=num_epochs,
num_workers=num_workers,
num_validation_samples=num_val,
num_test_samples=num_test,
data_processing_function=dl.get_sensor_bool_dryspot_runlevel,
import pickle
from pathlib import Path
import torch
import Resources.training as r
from Pipeline import torch_datagenerator as td
from Pipeline.data_gather import get_filelist_within_folder_blacklisted
from Pipeline.data_loader_dryspot import DataloaderDryspots
if __name__ == "__main__":
dlds = DataloaderDryspots(divide_by_100k=False)
batch_size = 131072
generator = td.LoopingDataGenerator(r.get_data_paths_base_0(),
get_filelist_within_folder_blacklisted,
dlds.get_sensor_bool_dryspot,
num_validation_samples=131072,
num_test_samples=1048576,
batch_size=batch_size,
split_load_path=r.dataset_split,
split_save_path=Path(),
num_workers=75,
looping_strategy=None)
all_sensor_inputs = []
for i, (inputs, _, _) in enumerate(generator):
all_sensor_inputs.append(inputs)
print(i)
all_sensor_values = torch.cat(all_sensor_inputs, dim=0)
_std = all_sensor_values.std(dim=0)
_mean = all_sensor_values.mean(dim=0)
print("Std\n", _std)
import torch
import Resources.training as r
from Models.erfh5_fullyConnected import S20DryspotModelFCWide
from Pipeline.data_gather import get_filelist_within_folder_blacklisted
from Pipeline.data_loader_dryspot import DataloaderDryspots
from Trainer.ModelTrainer import ModelTrainer
from Trainer.evaluation import BinaryClassificationEvaluator
from Utils.eval_utils import run_eval_w_binary_classificator
if __name__ == "__main__":
dl = DataloaderDryspots(sensor_indizes=((1, 8), (1, 8)),
aux_info=True)
checkpoint_p = r.chkp_S20_densenet_baseline_full_trainingset
adv_output_dir = checkpoint_p.parent / "advanced_eval"
m = ModelTrainer(
lambda: S20DryspotModelFCWide(),
data_source_paths=r.get_data_paths_base_0(),
save_path=r.save_path,
dataset_split_path=r.dataset_split,
cache_path=r.cache_path,
batch_size=32768,
train_print_frequency=100,
epochs=1000,
num_workers=75,
num_validation_samples=131072,
num_test_samples=1048576,
data_processing_function=dl.get_sensor_bool_dryspot,
data_gather_function=get_filelist_within_folder_blacklisted,
loss_criterion=torch.nn.BCELoss(),
from Utils.training_utils import read_cmd_params
if __name__ == "__main__":
"""
This is the second stage for training the FlowFrontNet: the DrySpotNet
with 1140 sensor data to Flowfront images.
Please add the path to the pretrained weights as parameter
checkpoint_path to the SensorDeconvToDryspotEfficient2 Model or use the command line option --checkpoint_path.
"""
args = read_cmd_params()
if args.demo is not None:
checkpoint_path = args.checkpoint_path
else:
checkpoint_path = "Use your own path to checkpoint."
dlds = DataloaderDryspots()
m = ModelTrainer(
lambda: SensorDeconvToDryspotEfficient2(pretrained="deconv_weights",
checkpoint_path=Path(
checkpoint_path),
freeze_nlayers=8),
data_source_paths=r.get_data_paths_base_0(),
save_path=r.save_path if args.demo is None else Path(args.demo),
load_datasets_path=r.datasets_dryspots,
cache_path=r.cache_path,
batch_size=2048,
train_print_frequency=100,
epochs=1000,
num_workers=75,
num_validation_samples=131072,
num_test_samples=1048576,
test_labels.extend(labels)
test_data = np.array(test_data)
test_labels = np.array(test_labels)
test_labels = np.ravel(test_labels)
print("Loaded Test data.")
return test_data, test_labels
if __name__ == "__main__":
num_samples = 150000
args = read_cmd_params()
print("Using ca. 150 000 samples.")
dl = DataloaderDryspots(sensor_indizes=((0, 1), (0, 1)))
print("Created Dataloader.")
generator = td.LoopingDataGenerator(
r.get_data_paths_base_0(),
get_filelist_within_folder_blacklisted,
dl.get_sensor_bool_dryspot,
batch_size=512,
num_validation_samples=131072,
num_test_samples=1048576,
split_load_path=r.datasets_dryspots,
split_save_path=r.save_path,
num_workers=75,
cache_path=r.cache_path,
cache_mode=td.CachingMode.Both,
dont_care_num_samples=False,
import torch
import Resources.training as r
from Models.erfh5_fullyConnected import S1140DryspotModelFCWide
from Pipeline.data_gather import get_filelist_within_folder_blacklisted
from Pipeline.data_loader_dryspot import DataloaderDryspots
from Trainer.ModelTrainer import ModelTrainer
from Trainer.evaluation import BinaryClassificationEvaluator
from Utils.eval_utils import run_eval_w_binary_classificator
if __name__ == "__main__":
dl = DataloaderDryspots(aux_info=True)
checkpoint_p = r.chkp_S1140_densenet_baseline_full_trainingset
adv_output_dir = checkpoint_p.parent / "advanced_eval"
m = ModelTrainer(
lambda: S1140DryspotModelFCWide(),
data_source_paths=r.get_data_paths_base_0(),
save_path=r.save_path,
dataset_split_path=r.dataset_split,
cache_path=r.cache_path,
batch_size=32768,
train_print_frequency=100,
epochs=1000,
num_workers=75,
num_validation_samples=131072,
num_test_samples=1048576,
data_processing_function=dl.get_sensor_bool_dryspot,
data_gather_function=get_filelist_within_folder_blacklisted,
loss_criterion=torch.nn.BCELoss(),
optimizer_function=lambda params: torch.optim.AdamW(params, lr=1e-4),
from pathlib import Path
import torch
import Resources.training as r
from Models.erfh5_ConvModel import DrySpotModel
from Pipeline.data_gather import get_filelist_within_folder_blacklisted
from Pipeline.data_loader_dryspot import DataloaderDryspots
from Trainer.ModelTrainer import ModelTrainer
from Trainer.evaluation import BinaryClassificationEvaluator
from Utils.training_utils import read_cmd_params
if __name__ == "__main__":
args = read_cmd_params()
dl = DataloaderDryspots(image_size=(143, 111))
m = ModelTrainer(
lambda: DrySpotModel(),
data_source_paths=r.get_data_paths_base_0(),
save_path=r.save_path,
load_datasets_path=r.datasets_dryspots,
cache_path=r.cache_path,
batch_size=2048,
train_print_frequency=100,
epochs=1000,
num_workers=75,
num_validation_samples=131072,
num_test_samples=1048576,
data_processing_function=dl.get_flowfront_bool_dryspot,
data_gather_function=get_filelist_within_folder_blacklisted,
from Utils.training_utils import read_cmd_params
if __name__ == "__main__":
"""
This is the second stage for training the FlowFrontNet: the DrySpotNet
with 80 sensor data to Flowfront images.
Please add the path to the pretrained weights as parameter
checkpoint_path to the SensorDeconvToDryspotEfficient2 Model or use the command line option --checkpoint_path.
"""
args = read_cmd_params()
if args.demo is not None:
checkpoint_path = args.checkpoint_path
else:
checkpoint_path = "Use your own path to checkpoint."
dl = DataloaderDryspots(sensor_indizes=((1, 4), (1, 4)))
m = ModelTrainer(
lambda: S80Deconv2ToDrySpotEff(demo_mode=True
if args.demo is not None else False,
pretrained="deconv_weights",
checkpoint_path=Path(checkpoint_path),
freeze_nlayers=9),
data_source_paths=r.get_data_paths_base_0(),
save_path=r.save_path if args.demo is None else Path(args.demo),
dataset_split_path=r.dataset_split,
cache_path=r.cache_path,
batch_size=2048,
train_print_frequency=100,
epochs=1000,
num_workers=75,
import Resources.training as r
from Models.erfh5_fullyConnected import S20DryspotModelFCWide
from Pipeline.data_gather import get_filelist_within_folder_blacklisted
from Pipeline.data_loader_dryspot import DataloaderDryspots
from Trainer.ModelTrainer import ModelTrainer
from Trainer.evaluation import BinaryClassificationEvaluator
from Utils.training_utils import read_cmd_params
if __name__ == "__main__":
"""
This is the starting point for training the feed foward network with 20 sensor data to binary classification.
"""
args = read_cmd_params()
dlds = DataloaderDryspots(sensor_indizes=((1, 8), (1, 8)))
m = ModelTrainer(
lambda: S20DryspotModelFCWide(demo_mode=True
if args.demo is not None else False),
data_source_paths=r.get_data_paths_base_0(),
save_path=r.save_path if args.demo is None else Path(args.demo),
load_datasets_path=r.datasets_dryspots,
cache_path=r.cache_path,
batch_size=32768,
train_print_frequency=100,
epochs=100,
num_workers=75,
num_validation_samples=131072,
num_test_samples=1048576,
data_processing_function=dlds.get_sensor_bool_dryspot,
data_gather_function=get_filelist_within_folder_blacklisted,
