def setUp(self):
self.training_save_path = resources.test_training_out_dir
self.training_data_paths = [
resources.test_training_src_dir / 'dry_spot_from_ff'
]
self.expected_num_epochs_during_training = 1
dlds = DataloaderDryspots(image_size=(143, 111),
ignore_useless_states=False)
self.dt = ModelTrainer(
lambda: DrySpotModel(),
data_source_paths=self.training_data_paths,
save_path=self.training_save_path,
batch_size=10,
epochs=self.expected_num_epochs_during_training,
num_validation_samples=5,
num_test_samples=5,
data_gather_function=dg.get_filelist_within_folder,
data_processing_function=dlds.get_flowfront_bool_dryspot,
loss_criterion=torch.nn.BCELoss(),
classification_evaluator_function=lambda summary_writer:
BinaryClassificationEvaluator(summary_writer=summary_writer,
save_path=self.training_save_path,
skip_images=True),
data_root=resources.test_src_dir,
)
def test_training_ok_notok(self):
dls = DataLoaderSensor()
model_trainer = ModelTrainer(
lambda: ERFH5_PressureSequence_Model(),
self.training_data_paths,
self.training_save_path,
epochs=self.expected_num_epochs_during_training,
data_gather_function=dg.get_filelist_within_folder,
data_processing_function=dls.sensorgrid_simulationsuccess,
num_validation_samples=1,
num_test_samples=1,
loss_criterion=torch.nn.BCELoss(),
classification_evaluator_function=lambda summary_writer:
BinaryClassificationEvaluator(summary_writer=summary_writer),
data_root=test_resources.test_src_dir,
)
model_trainer.start_training()
dirs = [e for e in self.training_save_path.iterdir() if e.is_dir()]
with open(dirs[0] / "output.log") as f:
content = f.read()
epochs = re.findall("Mean Loss on Eval", content)
self.assertTrue(len(epochs) > 0)
def test_training_load_optimizer(self):
dl = DataloaderImages((149, 117), ignore_useless_states=False)
st = ModelTrainer(
lambda: DeconvModelEfficient(),
self.test_src_dir,
self.training_save_path,
load_datasets_path=self.test_split_dir,
cache_path=None,
batch_size=16,
train_print_frequency=10,
epochs=self.expected_num_epochs_during_training,
num_workers=4,
num_validation_samples=2,
num_test_samples=self.num_test_samples,
data_processing_function=dl.get_sensordata_and_flowfront,
data_gather_function=get_filelist_within_folder_blacklisted,
loss_criterion=torch.nn.BCELoss(),
optimizer_path=self.checkpoint,
classification_evaluator_function=lambda summary_writer:
SensorToFlowfrontEvaluator(summary_writer=summary_writer),
data_root=test_resources.test_src_dir,
)
st.start_training()
after = len(st.optimizer.state.keys())
""" Optimizer has now more than 0 states, therefore was loaded """
self.assertGreater(after, 0)
def test_eval_preparation(self):
dl = DataloaderImages((149, 117), ignore_useless_states=False)
st = ModelTrainer(
lambda: DeconvModelEfficient(),
self.test_src_dir,
self.eval_output_path,
load_datasets_path=self.test_split_dir,
cache_path=None,
batch_size=2,
train_print_frequency=10,
epochs=0,
num_workers=4,
num_validation_samples=2,
num_test_samples=self.num_test_samples,
data_processing_function=dl.get_sensordata_and_flowfront,
data_gather_function=get_filelist_within_folder_blacklisted,
classification_evaluator_function=lambda summary_writer:
SensorToFlowfrontEvaluator(summary_writer=summary_writer),
data_root=test_resources.test_src_dir,
)
st.start_training()
dirs = [e for e in self.eval_output_path.iterdir() if e.is_dir()]
code_dir = dirs[0] / 'rtm-predictions'
slurm_script = dirs[0] / 'run_model_eval.sh'
self.assertTrue(os.path.isdir(code_dir))
self.assertTrue(os.path.isfile(slurm_script))
with open(slurm_script) as f:
lines = f.read().splitlines()
tokens = lines[-1].split()
self.assertEqual(dirs[0], Path(tokens[-3]))
st.writer.flush()
st.writer.close()
def test_training(self):
num_epochs = 2
dl = DataloaderImages((149, 117), ignore_useless_states=False)
st = ModelTrainer(
lambda: DeconvModelEfficient(),
self.test_src_dir,
self.training_save_path,
load_datasets_path=self.test_split_dir,
cache_path=None,
batch_size=16,
train_print_frequency=10,
epochs=num_epochs,
num_workers=4,
num_validation_samples=2,
num_test_samples=self.num_test_samples,
data_processing_function=dl.get_sensordata_and_flowfront,
data_gather_function=get_filelist_within_folder_blacklisted,
loss_criterion=torch.nn.BCELoss(),
classification_evaluator_function=lambda summary_writer:
SensorToFlowfrontEvaluator(summary_writer=summary_writer),
data_root=test_resources.test_src_dir,
)
st.start_training()
dirs = [e for e in self.training_save_path.iterdir() if e.is_dir()]
with open(dirs[0] / 'output.log') as f:
content = f.read()
epochs = re.findall('Mean Loss on Eval', content)
self.assertEqual(num_epochs, len(epochs))
# Check if steps are growing / if there are doubled steps in the output
steps = [
int(re.findall(r'\d+', x)[0])
for x in re.findall(r'Duration of step.+\d:', content)
]
self.assertEqual(len(set(steps)), len(steps))
def create_trainer_and_start(self,
out_path,
epochs=1,
load_test_set=False):
dlds = DataloaderFlowfrontSensor(sensor_indizes=((1, 8), (1, 8)))
m = ModelTrainer(
lambda: S20DryspotModelFCWide(),
data_source_paths=tr_resources.get_data_paths_debug(),
save_path=out_path,
load_datasets_path=self.torch_dataset_resources /
"reference_datasets",
cache_path=None,
num_validation_samples=8,
num_test_samples=8,
num_workers=0,
epochs=epochs,
data_processing_function=dlds.get_flowfront_sensor_bool_dryspot,
data_gather_function=dg.get_filelist_within_folder_blacklisted,
loss_criterion=torch.nn.BCELoss(),
optimizer_function=lambda params: torch.optim.AdamW(params,
lr=1e-4),
classification_evaluator_function=lambda summary_writer:
BinaryClassificationEvaluator(summary_writer=summary_writer),
load_test_set_in_training_mode=load_test_set,
data_root=test_resources.test_src_dir,
)
return m
def init_trainer():
dlds = DataloaderDryspots()
m = ModelTrainer(
lambda: model,
data_source_paths=filepaths,
save_path=save_path,
cache_path=cache_path,
batch_size=batch_size,
train_print_frequency=train_print_frequency,
looping_strategy=NoOpLoopingStrategy(),
epochs=epochs,
dummy_epoch=False,
num_workers=num_workers,
num_validation_samples=num_validation_samples,
num_test_samples=num_test_samples,
data_processing_function=dlds.
get_sensor_bool_dryspot_resized_matrix,
data_gather_function=get_filelist_within_folder_blacklisted,
data_root=data_root,
loss_criterion=torch.nn.BCELoss(),
optimizer_function=lambda params: torch.optim.AdamW(params,
lr=1e-4),
classification_evaluator_function=lambda:
BinaryClassificationEvaluator(),
lr_scheduler_function=lambda optim: ExponentialLR(optim, 0.5),
caching_torch=False,
demo_path=None,
hold_samples_in_memory=False,
)
return m
def test_eval(self):
dl = DataloaderImages((149, 117), ignore_useless_states=False)
st = ModelTrainer(
lambda: DeconvModelEfficient(),
self.test_src_dir,
self.eval_output_path,
load_datasets_path=self.test_split_dir,
cache_path=None,
batch_size=2,
train_print_frequency=10,
epochs=self.expected_num_epochs_during_training,
num_workers=10,
num_validation_samples=2,
num_test_samples=self.num_test_samples,
data_processing_function=dl.get_sensordata_and_flowfront,
data_gather_function=get_filelist_within_folder_blacklisted,
loss_criterion=torch.nn.BCELoss(),
classification_evaluator_function=lambda summary_writer:
SensorToFlowfrontEvaluator(self.eval_output_path /
"eval_on_test_set",
skip_images=False,
summary_writer=summary_writer),
data_root=test_resources.test_src_dir,
)
st.inference_on_test_set(
self.eval_output_path,
self.checkpoint,
classification_evaluator_function=lambda summary_writer:
SensorToFlowfrontEvaluator(self.eval_output_path /
"eval_on_test_set",
skip_images=False,
summary_writer=summary_writer))
with open(self.eval_output_path / "eval_on_test_set" /
"test_output.log") as f:
content = f.read()
loss = float(
re.findall(r'\d+.\d+',
re.findall(r'Eval: \d+\.\d+', content)[0])[0])
self.assertEqual(np.round(loss, 4), self.expected_loss)
img_path = self.eval_output_path / 'eval_on_test_set' / 'images'
list_all_imgs = list(img_path.glob('**/*.jpg'))
self.assertEqual(len(list_all_imgs), self.expected_num_frames)
class TestTrainingDryspotFF(unittest.TestCase):
def setUp(self):
self.training_save_path = resources.test_training_out_dir
self.training_data_paths = [
resources.test_training_src_dir / 'dry_spot_from_ff'
]
self.expected_num_epochs_during_training = 1
dlds = DataloaderDryspots(image_size=(143, 111),
ignore_useless_states=False)
self.dt = ModelTrainer(
lambda: DrySpotModel(),
data_source_paths=self.training_data_paths,
save_path=self.training_save_path,
batch_size=10,
epochs=self.expected_num_epochs_during_training,
num_validation_samples=5,
num_test_samples=5,
data_gather_function=dg.get_filelist_within_folder,
data_processing_function=dlds.get_flowfront_bool_dryspot,
loss_criterion=torch.nn.BCELoss(),
classification_evaluator_function=lambda summary_writer:
BinaryClassificationEvaluator(summary_writer=summary_writer,
save_path=self.training_save_path,
skip_images=True),
data_root=resources.test_src_dir,
)
def test_training(self):
self.dt.start_training()
dirs = [e for e in self.training_save_path.iterdir() if e.is_dir()]
with open(dirs[0] / "output.log") as f:
content = f.read()
epochs = re.findall("Mean Loss on Eval", content)
self.assertTrue(len(epochs) > 0)
def tearDown(self) -> None:
logging.shutdown()
r = logging.getLogger("")
[r.removeHandler(x) for x in r.handlers]
shutil.rmtree(self.training_save_path)
def test_save_load_training(self):
num_epochs = 2
dl = DataloaderImages((149, 117), ignore_useless_states=False)
st = ModelTrainer(
lambda: DeconvModelEfficient(),
self.test_src_dir,
self.training_save_path,
load_datasets_path=self.test_split_dir,
cache_path=None,
batch_size=16,
train_print_frequency=10,
epochs=num_epochs,
num_workers=4,
num_validation_samples=2,
num_test_samples=self.num_test_samples,
data_processing_function=dl.get_sensordata_and_flowfront,
data_gather_function=get_filelist_within_folder_blacklisted,
loss_criterion=torch.nn.BCELoss(),
classification_evaluator_function=lambda summary_writer:
SensorToFlowfrontEvaluator(summary_writer=summary_writer),
data_root=test_resources.test_src_dir,
)
st.start_training()
num_epochs = 2
dl = DataloaderImages((149, 117), ignore_useless_states=False)
st = ModelTrainer(
lambda: DeconvModelEfficient(),
self.test_src_dir,
self.training_save_path,
load_datasets_path=self.test_split_dir,
cache_path=None,
batch_size=16,
train_print_frequency=10,
epochs=num_epochs,
num_workers=4,
num_validation_samples=2,
num_test_samples=self.num_test_samples,
data_processing_function=dl.get_sensordata_and_flowfront,
data_gather_function=get_filelist_within_folder_blacklisted,
loss_criterion=torch.nn.BCELoss(),
classification_evaluator_function=lambda summary_writer:
SensorToFlowfrontEvaluator(summary_writer=summary_writer),
data_root=test_resources.test_src_dir,
)
st.start_training()
dlm = DataLoaderMesh(sensor_indices=((1, 2), (1, 2)))
mc = MeshCreator(batch_size)
mesh = mc.batched_mesh_torch(batch_size)
model = SensorMeshToFlowFrontModel(mesh, batch_size=batch_size)
m = ModelTrainer(
lambda: model,
data_source_paths=filepaths,
save_path=save_path,
cache_path=cache_path,
batch_size=batch_size,
train_print_frequency=train_print_frequency,
epochs=epochs,
dummy_epoch=True,
num_workers=num_workers,
num_validation_samples=num_validation_samples,
num_test_samples=num_test_samples,
data_processing_function=dlm.get_sensor_flowfront_mesh,
data_gather_function=get_filelist_within_folder_blacklisted,
data_root=data_root,
loss_criterion=torch.nn.BCELoss(),
optimizer_function=lambda params: torch.optim.AdamW(params, lr=1e-3),
classification_evaluator_function=lambda: MeshEvaluator(),
lr_scheduler_function=None,
caching_torch=False,
demo_path=None,
drop_last_batch=True)
m.start_training()
args = read_cmd_params()
img_size = (112, 96)
dl = DataloaderImages(image_size=img_size, sensor_indizes=((1, 4), (1, 4)))
m = ModelTrainer(
lambda: S80DeconvModelEfficient2(demo_mode=True
if args.demo is not None else False),
data_source_paths=r.get_data_paths_base_0(),
save_path=r.save_path if args.demo is None else Path(args.demo),
load_datasets_path=r.datasets_dryspots,
cache_path=r.cache_path,
batch_size=128,
train_print_frequency=100,
epochs=1000,
num_workers=75,
num_validation_samples=131072,
num_test_samples=1048576,
data_processing_function=dl.get_sensordata_and_flowfront,
data_gather_function=get_filelist_within_folder_blacklisted,
loss_criterion=torch.nn.MSELoss(),
optimizer_function=lambda params: torch.optim.AdamW(params, lr=0.0001),
classification_evaluator_function=lambda summary_writer:
SensorToFlowfrontEvaluator(summary_writer=summary_writer),
demo_path=args.demo,
run_eval_step_before_training=True,
resize_label_to=img_size if args.demo is not None else (0, 0))
if not args.run_eval:
m.start_training()
else:
m.inference_on_test_set(
from Trainer.evaluation import BinaryClassificationEvaluator
from Utils.eval_utils import run_eval_w_binary_classificator
if __name__ == "__main__":
dl = DataloaderDryspots(sensor_indizes=((1, 8), (1, 8)),
aux_info=True)
checkpoint_p = r.chkp_S20_densenet_baseline_full_trainingset
adv_output_dir = checkpoint_p.parent / "advanced_eval"
m = ModelTrainer(
lambda: S20DryspotModelFCWide(),
data_source_paths=r.get_data_paths_base_0(),
save_path=r.save_path,
dataset_split_path=r.dataset_split,
cache_path=r.cache_path,
batch_size=32768,
train_print_frequency=100,
epochs=1000,
num_workers=75,
num_validation_samples=131072,
num_test_samples=1048576,
data_processing_function=dl.get_sensor_bool_dryspot,
data_gather_function=get_filelist_within_folder_blacklisted,
loss_criterion=torch.nn.BCELoss(),
optimizer_function=lambda params: torch.optim.AdamW(params, lr=0.0001),
classification_evaluator_function=lambda: BinaryClassificationEvaluator(),
caching_torch=False
)
run_eval_w_binary_classificator(adv_output_dir, m, checkpoint_p)
else:
checkpoint_path = "Use your own path to checkpoint."
dlds = DataloaderDryspots()
m = ModelTrainer(
lambda: SensorDeconvToDryspotEfficient2(pretrained="deconv_weights",
checkpoint_path=Path(
checkpoint_path),
freeze_nlayers=8),
data_source_paths=r.get_data_paths_base_0(),
save_path=r.save_path if args.demo is None else Path(args.demo),
load_datasets_path=r.datasets_dryspots,
cache_path=r.cache_path,
batch_size=2048,
train_print_frequency=100,
epochs=1000,
num_workers=75,
num_validation_samples=131072,
num_test_samples=1048576,
data_processing_function=dlds.get_sensor_bool_dryspot,
data_gather_function=get_filelist_within_folder_blacklisted,
loss_criterion=torch.nn.BCELoss(),
optimizer_function=lambda params: torch.optim.AdamW(params, lr=1e-4),
classification_evaluator_function=lambda summary_writer:
BinaryClassificationEvaluator(summary_writer=summary_writer),
lr_scheduler_function=lambda optim: ExponentialLR(optim, 0.5),
demo_path=args.demo)
if not args.run_eval:
m.start_training()
else:
if __name__ == "__main__":
args = read_cmd_params()
dlds = DataloaderFlowfrontSensor(sensor_indizes=((0, 1), (0, 1)),
frame_count=1,
use_binary_sensor_only=True)
m = ModelTrainer(lambda: S1140DryspotModelFCWide(),
data_source_paths=r.get_data_paths_debug(),
save_path=r.save_path,
dataset_split_path=r.dataset_split,
cache_path=r.cache_path,
batch_size=2048,
train_print_frequency=100,
epochs=100,
num_workers=75,
num_validation_samples=512, # 131072,
num_test_samples=1024, # 1048576,
data_processing_function=dlds.get_flowfront_sensor_bool_dryspot,
data_gather_function=get_filelist_within_folder_blacklisted,
loss_criterion=torch.nn.BCELoss(),
optimizer_function=lambda params: torch.optim.AdamW(params, lr=1e-4),
classification_evaluator_function=lambda: BinaryClassificationEvaluator(),
dont_care_num_samples=True
)
if not args.run_eval:
m.start_training()
else:
m.inference_on_test_set(
output_path=Path(args.eval),
checkpoint_path=Path(args.checkpoint_path),
checkpoint_path = "Use your own path to checkpoint."
dl = DataloaderDryspots(sensor_indizes=((1, 4), (1, 4)))
m = ModelTrainer(
lambda: S80Deconv2ToDrySpotEff(demo_mode=True
if args.demo is not None else False,
pretrained="deconv_weights",
checkpoint_path=Path(checkpoint_path),
freeze_nlayers=9),
data_source_paths=r.get_data_paths_base_0(),
save_path=r.save_path if args.demo is None else Path(args.demo),
dataset_split_path=r.dataset_split,
cache_path=r.cache_path,
batch_size=2048,
train_print_frequency=100,
epochs=1000,
num_workers=75,
num_validation_samples=131072,
num_test_samples=1048576,
data_processing_function=dl.get_sensor_bool_dryspot,
data_gather_function=get_filelist_within_folder_blacklisted,
loss_criterion=torch.nn.MSELoss(),
optimizer_function=lambda params: torch.optim.AdamW(params, lr=0.0001),
classification_evaluator_function=lambda:
BinaryClassificationEvaluator(),
lr_scheduler_function=lambda optim: ExponentialLR(optim, 0.5),
demo_path=args.demo)
if not args.run_eval:
m.start_training()
else:
"""
This is the starting point for training the feed foward network with 80 sensor data to binary classification.
"""
args = read_cmd_params()
dlds = DataloaderDryspots(sensor_indizes=((1, 4), (1, 4)))
m = ModelTrainer(
lambda: S80DryspotModelFCWide(demo_mode=True if args.demo is not None else False),
data_source_paths=r.get_data_paths_base_0(),
save_path=r.save_path if args.demo is None else Path(args.demo),
load_datasets_path=r.datasets_dryspots,
cache_path=r.cache_path,
batch_size=32768,
train_print_frequency=100,
epochs=100,
num_workers=75,
num_validation_samples=131072,
num_test_samples=1048576,
data_processing_function=dlds.get_sensor_bool_dryspot,
data_gather_function=get_filelist_within_folder_blacklisted,
loss_criterion=torch.nn.BCELoss(),
optimizer_function=lambda params: torch.optim.AdamW(params, lr=1e-4),
classification_evaluator_function=lambda summary_writer:
BinaryClassificationEvaluator(summary_writer=summary_writer),
demo_path=args.demo
)
if not args.run_eval:
m.start_training()
else:
m.inference_on_test_set(
output_path=Path(args.eval),
import torch
import Resources.training as r
from Models.erfh5_pressuresequence_CRNN import ERFH5_PressureSequence_Model
from Pipeline import data_loader_sensor as dls, data_gather as dg
from Trainer.ModelTrainer import ModelTrainer
from Trainer.evaluation import BinaryClassificationEvaluator
if __name__ == "__main__":
data_source_paths = [r.data_root / "2019-07-24_16-32-40_5000p"]
save_path = r.save_path
cache_path = r.cache_path
trainer = ModelTrainer(
lambda: ERFH5_PressureSequence_Model(),
data_source_paths,
save_path,
None,
epochs=2,
data_gather_function=dg.get_filelist_within_folder,
data_processing_function=dls.sensorgrid_simulationsuccess,
loss_criterion=torch.nn.BCELoss(),
classification_evaluator_function=BinaryClassificationEvaluator(),
)
trainer.start_training()
print("training finished")
# def get_sampler(data_source):
# return RandomOverSampler(data_source, multiply_by=2)
m = ModelTrainer(
lambda: S80Deconv2ToDrySpotEff(pretrained="deconv_weights",
checkpoint_path=r.chkp_S80_to_ff2,
freeze_nlayers=9,
round_at=0.8),
data_source_paths=r.get_data_paths_base_0(),
save_path=r.save_path,
load_datasets_path=r.datasets_dryspots,
cache_path=r.cache_path,
batch_size=8192,
train_print_frequency=100,
epochs=1000,
num_workers=75,
num_validation_samples=131072,
num_test_samples=1048576,
data_processing_function=dl.get_sensor_bool_dryspot,
data_gather_function=get_filelist_within_folder_blacklisted,
loss_criterion=torch.nn.MSELoss(),
optimizer_function=lambda params: torch.optim.AdamW(params, lr=0.0001),
classification_evaluator_function=lambda summary_writer:
BinaryClassificationEvaluator(summary_writer=summary_writer),
# lr_scheduler_function=lambda optim: ExponentialLR(optim, 0.8),
# sampler=get_sampler
)
if not args.run_eval:
m.start_training()
else:
sensor_indizes=((1, 4), (1, 4)))
checkpoint_p = r.chkp_S80_to_ff2
adv_output_dir = checkpoint_p.parent / "advanced_eval"
m = ModelTrainer(
lambda: S80DeconvModelEfficient2(pretrained="all",
freeze_nlayers=9,
checkpoint_path=checkpoint_p,
round_at=.8),
data_source_paths=r.get_data_paths_base_0(),
save_path=r.save_path,
dataset_split_path=r.dataset_split,
cache_path=r.cache_path,
batch_size=2048,
train_print_frequency=100,
epochs=1000,
num_workers=75,
num_validation_samples=131072,
num_test_samples=1048576,
data_processing_function=dl.get_sensordata_and_flowfront,
data_gather_function=get_filelist_within_folder_blacklisted,
loss_criterion=torch.nn.MSELoss(),
optimizer_function=lambda params: torch.optim.AdamW(params, lr=0.0001),
classification_evaluator_function=lambda: SensorToFlowfrontEvaluator(),
)
output_path = r.chkp_S80_to_ff2.parent
m.inference_on_test_set(
output_path / "eval_on_test_set_rounded.5",
r.chkp_S80_to_ff2,
if __name__ == "__main__":
args = read_cmd_params()
dl = DataloaderFlowfrontSensor(image_size=(149, 117),
frame_count=1,
use_binary_sensor_only=True)
m = ModelTrainer(
lambda: DeconvModelEfficient(),
data_source_paths=r.get_data_paths_base_0(),
save_path=r.save_path,
dataset_split_path=r.dataset_split,
cache_path=r.cache_path,
batch_size=2048,
train_print_frequency=100,
epochs=100,
num_workers=75,
num_validation_samples=131072,
num_test_samples=1048576,
data_processing_function=dl.get_flowfront_sensor_and_flowfront_label,
data_gather_function=get_filelist_within_folder_blacklisted,
loss_criterion=torch.nn.MSELoss(),
optimizer_function=lambda params: torch.optim.AdamW(params, lr=1e-4),
classification_evaluator_function=lambda: SensorToFlowfrontEvaluator(),
dont_care_num_samples=True)
if not args.run_eval:
m.start_training()
else:
m.inference_on_test_set(
output_path=Path(args.eval),
checkpoint_path=Path(args.checkpoint_path),
if __name__ == "__main__":
args = read_cmd_params()
dl = DataloaderDryspots(image_size=(143, 111))
m = ModelTrainer(
lambda: DrySpotModel(),
data_source_paths=r.get_data_paths_base_0(),
save_path=r.save_path,
load_datasets_path=r.datasets_dryspots,
cache_path=r.cache_path,
batch_size=2048,
train_print_frequency=100,
epochs=1000,
num_workers=75,
num_validation_samples=131072,
num_test_samples=1048576,
data_processing_function=dl.get_flowfront_bool_dryspot,
data_gather_function=get_filelist_within_folder_blacklisted,
loss_criterion=torch.nn.MSELoss(),
optimizer_function=lambda params: torch.optim.AdamW(params, lr=0.0001),
classification_evaluator_function=lambda summary_writer:
BinaryClassificationEvaluator(summary_writer=summary_writer),
)
if not args.run_eval:
m.start_training()
else:
m.inference_on_test_set(
Path(args.eval),
mesh = dlm.get_batched_mesh_torch(batch_size, sample_file)
model = SensorMeshToDryspotResnet(mesh, batch_size=batch_size, weights_path=weights_path)
m = ModelTrainer(
lambda: model,
data_source_paths=filepaths,
dataset_split_path=dataset_split_path,
save_path=save_path,
cache_path=cache_path,
batch_size=batch_size,
train_print_frequency=train_print_frequency,
epochs=epochs,
dummy_epoch=True,
num_workers=num_workers,
num_validation_samples=num_validation_samples,
num_test_samples=num_test_samples,
data_processing_function=dlm.get_sensor_dryspot_mesh,
data_gather_function=get_filelist_within_folder_blacklisted,
data_root=data_root,
loss_criterion=torch.nn.BCELoss(),
optimizer_function=lambda params: torch.optim.SGD(params, lr=0.001),
classification_evaluator_function=lambda summary_writer:
BinaryClassificationEvaluator(summary_writer=summary_writer),
lr_scheduler_function=None,
caching_torch=False,
demo_path=None,
drop_last_batch=True
)
m.inference_on_test_set(checkpoint_path=checkpoint_path,
classification_evaluator_function=lambda summary_writer:
if __name__ == "__main__":
dl = DataloaderImages((149, 117))
checkpoint_p = r.chkp_S1140_to_ff_0_basepr
adv_output_dir = checkpoint_p.parent / "advanced_eval"
m = ModelTrainer(
lambda: DeconvModelEfficient(),
data_source_paths=r.get_data_paths_base_0(),
save_path=r.save_path,
load_datasets_path=r.datasets_dryspots,
cache_path=r.cache_path,
batch_size=2048,
train_print_frequency=10,
epochs=1000,
num_workers=75,
num_validation_samples=131072,
num_test_samples=1048576,
data_processing_function=dl.get_sensordata_and_flowfront,
data_gather_function=get_filelist_within_folder_blacklisted,
loss_criterion=torch.nn.MSELoss(),
optimizer_function=lambda params: torch.optim.AdamW(params, lr=0.0001),
classification_evaluator_function=lambda summary_writer:
SensorToFlowfrontEvaluator(summary_writer=summary_writer),
)
adv_output_dir.mkdir(exist_ok=True)
m.inference_on_test_set(
output_path=adv_output_dir,
checkpoint_path=checkpoint_p,
classification_evaluator_function=lambda summary_writer:
# model = SensorMeshToFlowFrontModelDGL(mesh, batch_size=batch_size)
m = ModelTrainer(
lambda: model,
data_source_paths=filepaths,
save_path=save_path,
cache_path=cache_path,
batch_size=batch_size,
train_print_frequency=train_print_frequency,
epochs=epochs,
dummy_epoch=False,
num_workers=num_workers,
num_validation_samples=num_validation_samples,
num_test_samples=num_test_samples,
data_processing_function=dlm.get_sensor_flowfront_mesh,
data_gather_function=get_filelist_within_folder_blacklisted,
looping_strategy=looping_strategy,
data_root=data_root,
loss_criterion=torch.nn.MSELoss(),
optimizer_function=lambda params: torch.optim.AdamW(params, lr=1e-4),
classification_evaluator_function=lambda:
FlowFrontMeshEvaluator(sample_file=sample_file,
save_path=save_path / "FF_Images/FF_272_loopingtest"),
lr_scheduler_function=None,
caching_torch=False,
demo_path=None,
drop_last_batch=True,
hold_samples_in_memory=False
)
m.start_training()
eval = True
dl = DataloaderImageSequences()
m = ModelTrainer(
lambda: OptimusPrime_c2D(batch_size)
if mode == "Transformer" else (FFTFF() if mode == "ConvLSTM" else
(FF2Perm_Baseline()
if mode == "Conv2D" else
(FF2Perm_3DConv()
if mode == "Conv3D" else FFTFF()))),
dataset_paths,
save_path,
cache_path=None,
batch_size=batch_size,
epochs=150,
num_workers=num_workers,
num_validation_samples=num_val,
num_test_samples=num_test,
data_processing_function=dl.get_flowfront_to_perm_map,
data_gather_function=get_filelist_within_folder_blacklisted,
loss_criterion=torch.nn.MSELoss(),
data_root=data_root,
demo_path=data_folder,
classification_evaluator_function=lambda: SensorToFlowfrontEvaluator(
skip_images=False,
ignore_inp=False,
sensors_shape=(143, 111),
save_path=save_path),
dummy_epoch=False)
if not eval:
m.start_training()
if __name__ == "__main__":
dl = DataloaderDryspots(sensor_indizes=((1, 4), (1, 4)),
aux_info=True)
checkpoint_p = r.chkp_S80_to_ds_thres
adv_output_dir = checkpoint_p.parent / "advanced_eval"
m = ModelTrainer(
lambda: S80Deconv2ToDrySpotEff(pretrained="all",
checkpoint_path=checkpoint_p,
freeze_nlayers=9,
round_at=.8),
data_source_paths=r.get_data_paths_base_0(),
save_path=r.save_path,
dataset_split_path=r.dataset_split,
cache_path=r.cache_path,
batch_size=2048,
train_print_frequency=100,
epochs=1000,
num_workers=75,
num_validation_samples=131072,
num_test_samples=1048576,
data_processing_function=dl.get_sensor_bool_dryspot,
data_gather_function=get_filelist_within_folder_blacklisted,
loss_criterion=torch.nn.MSELoss(),
optimizer_function=lambda params: torch.optim.AdamW(params, lr=0.0001),
classification_evaluator_function=lambda: BinaryClassificationEvaluator(),
caching_torch=False
)
run_eval_w_binary_classificator(adv_output_dir, m, checkpoint_p)
batch_size = 256
dataset_paths = r.get_regular_sampled_data_paths()
num_workers = 16
num_val = 100
num_test = 100
data_root = r.data_root_every_step
dl = DataloaderImageSequences()
m = ModelTrainer(
lambda: Bumblebee2(),
dataset_paths,
r.save_path,
cache_path=r.cache_path,
batch_size=batch_size,
epochs=150,
num_workers=num_workers,
num_validation_samples=num_val,
num_test_samples=num_test,
data_processing_function=dl.get_flowfront_to_flowfront,
data_gather_function=get_filelist_within_folder_blacklisted,
loss_criterion=torch.nn.MSELoss(),
data_root=data_root,
classification_evaluator_function=lambda: None,
dummy_epoch=False)
if not args.eval:
m.start_training()
else:
m.inference_on_test_set(
Path(args.eval),
Path(args.checkpoint_path),
lambda summary_writer: SensorToFlowfrontEvaluator(
dl = DataloaderDryspots(sensor_indizes=sensor_indices[str(num_sensors)],
aux_info=True,
image_size=(143, 111))
m = ModelTrainer(
lambda: SensorToBinaryRunwiseModel(num_sensors, conv_lstm_sizes,
fc_sizes),
dataset_paths,
r.save_path,
dataset_split_path=None, # r.dataset_split,
data_root=data_root,
cache_path=r.cache_path,
batch_size=batch_size,
epochs=num_epochs,
num_workers=num_workers,
num_validation_samples=num_val,
num_test_samples=num_test,
data_processing_function=dl.get_sensor_bool_dryspot_runlevel,
data_gather_function=get_filelist_within_folder_blacklisted,
loss_criterion=torch.nn.BCELoss(),
optimizer_function=lambda params: torch.optim.AdamW(params, lr=lr),
classification_evaluator_function=lambda:
BinaryClassificationEvaluator(skip_images=not create_data_plots,
max_epochs=num_epochs,
data_loader=dl),
# lr_scheduler_function=lambda optim: ExponentialLR(optim, 0.5),
dummy_epoch=False,
caching_torch=use_cache,
run_name=run_name,
save_in_mlflow_directly=True)
if not args.run_eval:
m.start_training()
if __name__ == "__main__":
args = read_cmd_params()
dlds = DataloaderFlowfrontSensor(sensor_indizes=((1, 8), (1, 8)))
m = ModelTrainer(
lambda: S20DryspotModelFCWide(),
data_source_paths=r.get_data_paths_base_0(),
save_path=r.save_path,
load_datasets_path=r.datasets_dryspots,
cache_path=r.cache_path,
batch_size=2048,
train_print_frequency=100,
epochs=1000,
num_workers=75,
num_validation_samples=131072,
num_test_samples=1048576,
data_processing_function=dlds.get_flowfront_sensor_bool_dryspot,
data_gather_function=get_filelist_within_folder_blacklisted,
loss_criterion=torch.nn.BCELoss(),
optimizer_function=lambda params: torch.optim.AdamW(params, lr=1e-4),
classification_evaluator_function=lambda summary_writer:
BinaryClassificationEvaluator(summary_writer=summary_writer),
save_torch_dataset_path=r.datasets_dryspots_torch /
Path(__file__).stem,
load_torch_dataset_path=r.datasets_dryspots_torch /
Path(__file__).stem,
# lr_scheduler_function=lambda optim: ExponentialLR(optim, 0.1),
)
if not args.run_eval:
m.start_training()
else:
