def test_for_leakage(self): """ Simple test to check that raw target data does NOT leak during validation steps. """ src, trg = next(iter(self.validation_loader)) trg_mem = trg.clone().detach() result = greedy_decode(self.model, src, 20, trg) self.assertNotEqual(result[0, 1, 0], trg_mem[0, 1, 0]) self.assertEqual(result[0, 1, 1], trg_mem[0, 1, 1]) self.assertEqual(result[0, 1, 2], trg_mem[0, 1, 2]) loss = pytorch_criterion_dict["MSE"]()(trg, trg_mem) self.assertNotEqual(result[0, 1, 0], result[0, 4, 0]) self.assertGreater(loss, 0)
def test_for_leakage(self): """ Simple test to check that raw target data does NOT leak during validation. """ src_mask = generate_square_subsequent_mask(self.sequence_size) src, trg = next(iter(self.validation_loader)) trg_mem = trg.clone().detach() result = greedy_decode(self.model, src, 20, trg) self.assertNotEqual(result[0, 1, 0], trg_mem[0, 1, 0]) self.assertEqual(result[0, 1, 1], trg_mem[0, 1, 1]) self.assertEqual(result[0, 1, 2], trg_mem[0, 1, 2]) loss = pytorch_criterion_dict["MSE"](trg, trg_mem) #self.assertNotAlmostEqual(result[0, 1, 0], result[0, 1, 2]) self.assertNotEqual(result[0,1,0], result[0, 4, 0]) self.assertGreater(loss, 0)
def compute_validation(validation_loader: DataLoader, model, epoch: int, sequence_size: int, criterion: Type[torch.nn.modules.loss._Loss], device: torch.device, decoder_structure=False, meta_data_model=None, use_wandb: bool = False, meta_model=None, multi_targets=1, val_or_test="validation_loss", probabilistic=False) -> float: """Function to compute the validation loss metrics :param validation_loader: The data-loader of either validation or test-data :type validation_loader: DataLoader :param model: model :type model: [type] :param epoch: The epoch where the validation/test loss is being computed. :type epoch: int :param sequence_size: [description] :type sequence_size: int :param criterion: [description] :type criterion: Type[torch.nn.modules.loss._Loss] :param device: The device :type device: torch.device :param decoder_structure: Whether the model should use sequential decoding, defaults to False :type decoder_structure: bool, optional :param meta_data_model: The model to handle the meta-data, defaults to None :type meta_data_model: PyTorchForecast, optional :param use_wandb: Whether Weights and Biases is in use, defaults to False :type use_wandb: bool, optional :param meta_model: Whether the model leverages meta-data, defaults to None :type meta_model: bool, optional :param multi_targets: Whether the model, defaults to 1 :type multi_targets: int, optional :param val_or_test: Whether validation or test loss is computed, defaults to "validation_loss" :type val_or_test: str, optional :param probabilistic: Whether the model is probablistic, defaults to False :type probabilistic: bool, optional :return: The loss of the first metric in the list. :rtype: float """ print('Computing validation loss') unscaled_crit = dict.fromkeys(criterion, 0) scaled_crit = dict.fromkeys(criterion, 0) model.eval() output_std = None multi_targs1 = multi_targets scaler = None if validation_loader.dataset.no_scale: scaler = validation_loader.dataset with torch.no_grad(): i = 0 loss_unscaled_full = 0.0 for src, targ in validation_loader: src = src if isinstance(src, list) else src.to(device) targ = targ if isinstance(targ, list) else targ.to(device) # targ = targ if isinstance(targ, list) else targ.to(device) i += 1 if decoder_structure: if type(model).__name__ == "SimpleTransformer": targ_clone = targ.detach().clone() output = greedy_decode(model, src, targ.shape[1], targ_clone, device=device)[:, :, 0] elif type(model).__name__ == "Informer": multi_targets = multi_targs1 filled_targ = targ[1].clone() pred_len = model.pred_len filled_targ[:, -pred_len:, :] = torch.zeros_like( filled_targ[:, -pred_len:, :]).float().to(device) output = model(src[0].to(device), src[1].to(device), filled_targ.to(device), targ[0].to(device)) labels = targ[1][:, -pred_len:, 0:multi_targets] src = src[0] multi_targets = False else: output = simple_decode(model=model, src=src, max_seq_len=targ.shape[1], real_target=targ, output_len=sequence_size, multi_targets=multi_targets, probabilistic=probabilistic, scaler=scaler) if probabilistic: output, output_std = output[0], output[1] output, output_std = output[:, :, 0], output_std[0] output_dist = torch.distributions.Normal( output, output_std) else: if probabilistic: output_dist = model(src.float()) output = output_dist.mean.detach().numpy() output_std = output_dist.stddev.detach().numpy() else: output = model(src.float()) if multi_targets == 1: labels = targ[:, :, 0] elif multi_targets > 1: labels = targ[:, :, 0:multi_targets] validation_dataset = validation_loader.dataset for crit in criterion: if validation_dataset.scale: # Should this also do loss.item() stuff? if len(src.shape) == 2: src = src.unsqueeze(0) src1 = src[:, :, 0:multi_targets] loss_unscaled_full = compute_loss(labels, output, src1, crit, validation_dataset, probabilistic, output_std, m=multi_targets) unscaled_crit[crit] += loss_unscaled_full.item() * len( labels.float()) loss = compute_loss(labels, output, src, crit, False, probabilistic, output_std, m=multi_targets) scaled_crit[crit] += loss.item() * len(labels.float()) if use_wandb: if loss_unscaled_full: scaled = { k.__class__.__name__: v / (len(validation_loader.dataset) - 1) for k, v in scaled_crit.items() } newD = { k.__class__.__name__: v / (len(validation_loader.dataset) - 1) for k, v in unscaled_crit.items() } wandb.log({ 'epoch': epoch, val_or_test: scaled, "unscaled_" + val_or_test: newD }) else: scaled = { k.__class__.__name__: v / (len(validation_loader.dataset) - 1) for k, v in scaled_crit.items() } wandb.log({'epoch': epoch, val_or_test: scaled}) model.train() return list(scaled_crit.values())[0]
def compute_validation(validation_loader: DataLoader, model, epoch: int, sequence_size: int, criterion: Type[torch.nn.modules.loss._Loss], device: torch.device, decoder_structure=False, use_wandb: bool = False, val_or_test="validation_loss") -> float: """ Function to compute the validation or test loss """ model.eval() loop_loss = 0.0 with torch.no_grad(): i = 0 loss_unscaled_full = 0.0 for src, targ in validation_loader: src = src.to(device) targ = targ.to(device) i += 1 if decoder_structure: if type(model).__name__ == "SimpleTransformer": targ_clone = targ.detach().clone() output = greedy_decode(model, src, targ.shape[1], targ_clone, device=device)[:, :, 0] else: output = simple_decode(model, src, targ.shape[1], targ, 1)[:, :, 0] else: output = model(src.float()) labels = targ[:, :, 0] validation_dataset = validation_loader.dataset if validation_dataset.scale: # unscaled_src = validation_dataset.scale.inverse_transform(src.cpu()) unscaled_out = validation_dataset.inverse_scale(output.cpu()) unscaled_labels = validation_dataset.inverse_scale( labels.cpu()) loss_unscaled = criterion(unscaled_out, unscaled_labels.float()) loss_unscaled_full += len( labels.float()) * loss_unscaled.item() if i % 10 == 0 and use_wandb: import wandb wandb.log({ "trg": unscaled_labels, "model_pred": unscaled_out }) loss = criterion(output, labels.float()) loop_loss += len(labels.float()) * loss.item() if use_wandb: import wandb if loss_unscaled_full: tot_unscaled_loss = loss_unscaled_full / ( len(validation_loader.dataset) - 1) wandb.log({ 'epoch': epoch, val_or_test: loop_loss / (len(validation_loader.dataset) - 1), "unscaled_" + val_or_test: tot_unscaled_loss }) else: wandb.log({ 'epoch': epoch, val_or_test: loop_loss / (len(validation_loader.dataset) - 1) }) model.train() return loop_loss / (len(validation_loader.dataset) - 1)
def compute_validation(validation_loader: DataLoader, model, epoch: int, sequence_size: int, criterion: Type[torch.nn.modules.loss._Loss], device: torch.device, decoder_structure=False, meta_data_model=None, use_wandb: bool = False, meta_model=None, val_or_test="validation_loss", probabilistic=False) -> float: """ Function to compute the validation or the test loss """ print('compute_validation') model.eval() loop_loss = 0.0 output_std = None with torch.no_grad(): i = 0 loss_unscaled_full = 0.0 for src, targ in validation_loader: src = src.to(device) targ = targ.to(device) i += 1 if decoder_structure: if type(model).__name__ == "SimpleTransformer": targ_clone = targ.detach().clone() output = greedy_decode(model, src, targ.shape[1], targ_clone, device=device)[:, :, 0] else: if probabilistic: output, output_std = simple_decode( model, src, targ.shape[1], targ, 1, probabilistic=probabilistic) output, output_std = output[:, :, 0], output_std[0] output_dist = torch.distributions.Normal( output, output_std) else: output = simple_decode( model=model, src=src, max_seq_len=targ.shape[1], real_target=targ, output_len=1, probabilistic=probabilistic)[:, :, 0] else: if probabilistic: output_dist = model(src.float()) output = output_dist.mean.detach().numpy() output_std = output_dist.stddev.detach().numpy() else: output = model(src.float()) labels = targ[:, :, 0] validation_dataset = validation_loader.dataset if validation_dataset.scale: loss_unscaled_full += compute_loss(labels, output, src, criterion, validation_dataset, probabilistic, output_std) loss = compute_loss(labels, output, src, criterion, False, probabilistic, output_std) loop_loss += len(labels.float()) * loss.item() if use_wandb: if loss_unscaled_full: tot_unscaled_loss = loss_unscaled_full / ( len(validation_loader.dataset) - 1) wandb.log({ 'epoch': epoch, val_or_test: loop_loss / (len(validation_loader.dataset) - 1), "unscaled_" + val_or_test: tot_unscaled_loss }) else: wandb.log({ 'epoch': epoch, val_or_test: loop_loss / (len(validation_loader.dataset) - 1) }) model.train() return loop_loss / (len(validation_loader.dataset) - 1)
def compute_validation(validation_loader: DataLoader, # s lint model, epoch: int, sequence_size: int, criterion: Type[torch.nn.modules.loss._Loss], device: torch.device, decoder_structure=False, meta_data_model=None, use_wandb: bool = False, meta_model=None, val_or_test="validation_loss", probabilistic=False) -> float: """ Function to compute the validation or the test loss """ print('compute_validation') model.eval() loop_loss = 0.0 with torch.no_grad(): i = 0 loss_unscaled_full = 0.0 for src, targ in validation_loader: src = src.to(device) targ = targ.to(device) i += 1 if decoder_structure: if type(model).__name__ == "SimpleTransformer": targ_clone = targ.detach().clone() output = greedy_decode( model, src, targ.shape[1], targ_clone, device=device)[ :, :, 0] else: if probabilistic: output, output_std = simple_decode(model, src, targ.shape[1], targ, 1, probabilistic=probabilistic) output, output_std = output[:, :, 0], output_std[0] output_dist = torch.distributions.Normal(output, output_std) else: output = simple_decode(model=model, src=src, max_seq_len=targ.shape[1], real_target=targ, output_len=1, probabilistic=probabilistic)[:, :, 0] else: if probabilistic: output_dist = model(src.float()) output = output_dist.mean.detach().numpy() output_std = output_dist.stddev.detach().numpy() else: output = model(src.float()) labels = targ[:, :, 0] validation_dataset = validation_loader.dataset if validation_dataset.scale: unscaled_labels = validation_dataset.inverse_scale(labels) if probabilistic: unscaled_out = validation_dataset.inverse_scale(output) try: output_std = numpy_to_tvar(output_std) except Exception: pass unscaled_dist = torch.distributions.Normal(unscaled_out, output_std) loss_unscaled = -unscaled_dist.log_prob(unscaled_labels.float()).sum() # FIX THIS loss_unscaled_full += len(labels.float()) * loss_unscaled.numpy().item() else: # unscaled_src = validation_dataset.scale.inverse_transform(src.cpu()) unscaled_out = validation_dataset.inverse_scale(output.cpu()) unscaled_labels = validation_dataset.inverse_scale(labels.cpu()) loss_unscaled = criterion(unscaled_out, unscaled_labels.float()) loss_unscaled_full += len(labels.float()) * loss_unscaled.item() if i % 10 == 0 and use_wandb: wandb.log({"trg": unscaled_labels, "model_pred": unscaled_out}) if probabilistic: loss = -output_dist.log_prob(labels.float()).sum() # FIX THIS loss = loss.numpy() elif isinstance(criterion, GaussianLoss): g_loss = GaussianLoss(output[0], output[1]) loss = g_loss(labels) else: loss = criterion(output, labels.float()) loop_loss += len(labels.float()) * loss.item() if use_wandb: if loss_unscaled_full: tot_unscaled_loss = loss_unscaled_full / (len(validation_loader.dataset) - 1) wandb.log({'epoch': epoch, val_or_test: loop_loss / (len(validation_loader.dataset) - 1), "unscaled_" + val_or_test: tot_unscaled_loss}) else: wandb.log({'epoch': epoch, val_or_test: loop_loss / (len(validation_loader.dataset) - 1)}) model.train() return loop_loss / (len(validation_loader.dataset) - 1)
def compute_validation(validation_loader: DataLoader, model, epoch: int, sequence_size: int, criterion: Type[torch.nn.modules.loss._Loss], device: torch.device, decoder_structure=False, meta_data_model=None, use_wandb: bool = False, meta_model=None, multi_targets=1, val_or_test="validation_loss", probabilistic=False) -> float: """Function to compute the validation loss metrics :param validation_loader: The data-loader of either validation or test-data :type validation_loader: DataLoader :param model: model :type model: [type] :param epoch: [description] :type epoch: int :param sequence_size: [description] :type sequence_size: int :param criterion: [description] :type criterion: Type[torch.nn.modules.loss._Loss] :param device: [description] :type device: torch.device :param decoder_structure: [description], defaults to False :type decoder_structure: bool, optional :param meta_data_model: [description], defaults to None :type meta_data_model: [type], optional :param use_wandb: [description], defaults to False :type use_wandb: bool, optional :param meta_model: [description], defaults to None :type meta_model: [type], optional :param multi_targets: [description], defaults to 1 :type multi_targets: int, optional :param val_or_test: [description], defaults to "validation_loss" :type val_or_test: str, optional :param probabilistic: [description], defaults to False :type probabilistic: bool, optional :return: [description] :rtype: float """ print('compute_validation') unscaled_crit = dict.fromkeys(criterion, 0) scaled_crit = dict.fromkeys(criterion, 0) model.eval() output_std = None scaler = None if validation_loader.dataset.no_scale: scaler = validation_loader.dataset with torch.no_grad(): i = 0 loss_unscaled_full = 0.0 for src, targ in validation_loader: src = src.to(device) targ = targ.to(device) i += 1 if decoder_structure: if type(model).__name__ == "SimpleTransformer": targ_clone = targ.detach().clone() output = greedy_decode( model, src, targ.shape[1], targ_clone, device=device)[ :, :, 0] else: if probabilistic: output, output_std = simple_decode(model, src, targ.shape[1], targ, 1, multi_targets=multi_targets, probabilistic=probabilistic, scaler=scaler) output, output_std = output[:, :, 0], output_std[0] output_dist = torch.distributions.Normal(output, output_std) else: output = simple_decode(model=model, src=src, max_seq_len=targ.shape[1], real_target=targ, output_len=sequence_size, multi_targets=multi_targets, probabilistic=probabilistic, scaler=scaler)[:, :, 0:multi_targets] else: if probabilistic: output_dist = model(src.float()) output = output_dist.mean.detach().numpy() output_std = output_dist.stddev.detach().numpy() else: output = model(src.float()) if multi_targets == 1: labels = targ[:, :, 0] elif multi_targets > 1: labels = targ[:, :, 0:multi_targets] validation_dataset = validation_loader.dataset for crit in criterion: if validation_dataset.scale: # Should this also do loss.item() stuff? if len(src.shape) == 2: src = src.unsqueeze(0) src1 = src[:, :, 0:multi_targets] loss_unscaled_full = compute_loss(labels, output, src1, crit, validation_dataset, probabilistic, output_std, m=multi_targets) unscaled_crit[crit] += loss_unscaled_full.item() * len(labels.float()) loss = compute_loss(labels, output, src, crit, False, probabilistic, output_std, m=multi_targets) scaled_crit[crit] += loss.item() * len(labels.float()) if use_wandb: if loss_unscaled_full: scaled = {k.__class__.__name__: v / (len(validation_loader.dataset) - 1) for k, v in scaled_crit.items()} newD = {k.__class__.__name__: v / (len(validation_loader.dataset) - 1) for k, v in unscaled_crit.items()} wandb.log({'epoch': epoch, val_or_test: scaled, "unscaled_" + val_or_test: newD}) else: scaled = {k.__class__.__name__: v / (len(validation_loader.dataset) - 1) for k, v in scaled_crit.items()} wandb.log({'epoch': epoch, val_or_test: scaled}) model.train() return list(scaled_crit.values())[0]