class small_runs:
name = "1e5"
with_outliers = analysis.TrainingStability("Ribbles_w_outliers_1e5", None)
with_outliers.load()
without_outliers = analysis.TrainingStability("Ribbles_wo_outliers_1e5",
None)
without_outliers.load()
interactive.save_value("ensemble size 1e5 with outliers",
len(with_outliers.runs), ".1e")
interactive.save_value("ensemble size 1e5 without outliers",
len(without_outliers.runs), ".1e")
] #[7:-3] #np.logspace(9.3,11,4)/dl_config['batch_size']
latent_dims = [2, 3, 4, 6]
samples_per_time = 1 #5
time_per_batch = 16 / (1e8 / dl_config['batch_size'])
print(
"estimated runtime in hrs:",
samples_per_time * len(latent_dims) * np.sum(training_batches) *
time_per_batch / 60 / 60)
#%%
while (True):
for count_batches in tqdm(training_batches, "Progress", leave=True):
for dim in latent_dims:
model_config = {"latent_size": dim}
gym_factory = lambda: Gym(
training.best_model_class(model_config),
device,
EMD.torch_auto,
train,
val,
verbose=False,
)
r = analysis.TrainingStability(
f"corrected training time vs outliers",
gym_factory,
batches=int(count_batches),
auc_classes=val_classes,
add_attributes=model_config)
r.run(samples_per_time)
print("-----------Done, restarting task...")
transform=SingleWaveformPreprocessing(),
MC_types=0, # only train on valid events
)
dataset_val = MCDataset(filename=filename,
key="val/waveforms",
transform=SingleWaveformPreprocessing())
train_wo_outliers = DataLoader(dataset_train_wo_outliers, **dl_config)
val = DataLoader(dataset_val, batch_size=1024, num_workers=12)
val_classes = dataset_val.df.MC_type
#%%
while True:
for name, batches in zip(["1e5", "1e6"], [1000, 10000]):
gym_factory = lambda: Gym(
training.best_model_class(),
device,
EMD.torch_auto,
train_wo_outliers,
val,
verbose=False,
)
interactive.save_value(
"waveforms used for training wo outliers " + name,
batches * gym_factory().data_train.batch_size,
)
r = analysis.TrainingStability("Ribbles_wo_outliers_" + name,
gym_factory,
batches=batches,
auc_classes=val_classes)
r.run(20)
filename = config.root + config.MC.filename
dataset_val_toy = MCDataset(filename=filename,
key="val/waveforms",
transform=SingleWaveformPreprocessing())
val_toy = DataLoader(dataset_val_toy, batch_size=1024, num_workers=12)
dataset_val_MC = SingleWaveformDataset(
load_waveform_only=False,
transform=SingleWaveformPreprocessing(),
batch_loading_size=1024,
split=1,
)
val_MC = DataLoader(dataset_val_MC, batch_size=12, num_workers=1)
#%%
normal_toy_runs = analysis.TrainingStability("Ribbles_w_outliers_1e6", None)
normal_toy_runs.load()
#%%
best_loss = np.argmin(normal_toy_runs.loss_train)
best_model = normal_toy_runs.model[best_loss]
model_toy = ConvAE()
model_toy.load_state_dict(best_model.to_dict())
model_toy.eval()
interactive.save_value("auc of best toy model on toy val",
normal_toy_runs.auc[best_loss])
#%%
model_MC = ConvAE({"latent_size": 6})
model_MC.load_state_dict(
import icae.interactive_setup as interactive
import icae.results.n01_toy.n03_mode_training as training
import icae.tools.loss.EMD as EMD
from icae.models.waveform.simple import ConvAE
from icae.tools import analysis
from icae.tools.config_loader import config
from icae.tools.dataset.MC import MCDataset
from icae.tools.dataset.single import SingleWaveformPreprocessing
from icae.tools.torch.gym import Gym
plt.set_plot_path(__file__)
interactive.set_saved_value_yaml_root_by_filename(__file__)
training_set_size = 1e6 # TODO: read from yaml
# %%
r = analysis.TrainingStability("corrected training time vs outliers", None)
r.load()
# %%
times = np.unique(r.count_training_waveforms)
actual_times = []
latents = np.unique(r.latent_size)
time_index = [np.where(r.count_training_waveforms == t) for t in times]
latent_index = [np.where(r.latent_size == s) for s in latents]
# %%
aucs_mean = np.zeros([len(times), len(latents)]) * np.NaN
aucs_best = np.zeros_like(aucs_mean) * np.NaN
aucs_std = np.zeros_like(aucs_mean) * np.NaN
val_loss_mean = np.zeros_like(aucs_mean) * np.NaN
val_loss_mean_std = np.zeros_like(aucs_mean) * np.NaN
train_loss_mean = np.zeros_like(aucs_mean) * np.NaN