def compute_loss(labels, output, src, criterion, validation_dataset, probabilistic=None, output_std=None, m=1): # Warning this assumes src target is 1-D if not probabilistic and isinstance(output, torch.Tensor): if len(labels.shape) != len(output.shape): print(labels.shape) print(output.shape) if len(labels.shape) > 1: if labels.shape[1] == output.shape[1]: labels = labels.unsqueeze(2) else: labels = labels.unsqueeze(0) if probabilistic: if type(output_std) != torch.Tensor: print("Converted") output_std = torch.from_numpy(output_std) if type(output) != torch.Tensor: output = torch.from_numpy(output) output_dist = torch.distributions.Normal(output, output_std) if validation_dataset: if probabilistic: unscaled_out = validation_dataset.inverse_scale(output) try: output_std = numpy_to_tvar(output_std) except Exception: pass output_dist = torch.distributions.Normal(unscaled_out, output_std) else: output = validation_dataset.inverse_scale(output.cpu()) labels = validation_dataset.inverse_scale(labels.cpu()) src = validation_dataset.inverse_scale(src.cpu()) if probabilistic: loss = -output_dist.log_prob(labels.float()).sum() # FIX THIS loss = loss.numpy() elif isinstance(criterion, GaussianLoss): g_loss = GaussianLoss(output[0], output[1]) loss = g_loss(labels) elif isinstance(criterion, MASELoss): assert len(labels.shape) == len(output.shape) loss = criterion(labels.float(), output, src, m) else: assert len(labels.shape) == len(output.shape) assert labels.shape[0] == output.shape[0] loss = criterion(output, labels.float()) return loss
def torch_single_train(model: PyTorchForecast, opt: optim.Optimizer, criterion: Type[torch.nn.modules.loss._Loss], data_loader: DataLoader, takes_target: bool, meta_data_model: PyTorchForecast, meta_data_model_representation: torch.Tensor, forward_params: Dict = {}) -> float: i = 0 running_loss = 0.0 for src, trg in data_loader: opt.zero_grad() # Convert to CPU/GPU/TPU src = src.to(model.device) trg = trg.to(model.device) # TODO figure how to avoid if meta_data_model: representation = meta_data_model.model.generate_representation( meta_data_model_representation) forward_params["meta_data"] = representation if takes_target: forward_params["t"] = trg output = model.model(src, **forward_params) labels = trg[:, :, 0] if isinstance(criterion, GaussianLoss): g_loss = GaussianLoss(output[0], output[1]) loss = g_loss(labels) else: loss = criterion(output, labels.float()) # TODO fix Guassian loss if loss > 100: print("Warning: high loss detected") loss.backward() opt.step() if torch.isnan(loss) or loss == float('inf'): raise ValueError( "Error infinite or NaN loss detected. Try normalizing data or performing interpolation" ) running_loss += loss.item() i += 1 print("The running loss is:") print(running_loss) print("The number of items in train is: ") print(i) total_loss = running_loss / float(i) return total_loss
def compute_loss(labels, output, src, criterion, validation_dataset, probabilistic=None, output_std=None, m=1): """Function for computing the loss :param labels: The real values for the target. Shape can be variable but should follow (batch_size, time) :type labels: torch.Tensor :param output: The output of the model :type output: torch.Tensor :param src: The source values (only really needed for the MASELoss function) :type src: torch.Tensor :param criterion: [description] :type criterion: [type] :param validation_dataset: Only passed when unscaling of data is needed. :type validation_dataset: torch.utils.data.dataset :param probabilistic: Whether the model is a probabalistic returns a distribution, defaults to None :type probabilistic: [type], optional :param output_std: The standard distribution, defaults to None :type output_std: [type], optional :param m: The number of targs defaults to 1 :type m: int, optional :return: Returns the computed loss :rtype: float """ if isinstance(criterion, GaussianLoss): if len(output[0].shape) > 2: g_loss = GaussianLoss(output[0][:, :, 0], output[1][:, :, 0]) else: g_loss = GaussianLoss(output[0][:, 0], output[1][:, 0]) loss = g_loss(labels) return loss if not probabilistic and isinstance(output, torch.Tensor): if len(labels.shape) != len(output.shape): if len(labels.shape) > 1: if labels.shape[1] == output.shape[1]: labels = labels.unsqueeze(2) else: labels = labels.unsqueeze(0) if probabilistic: if type(output_std) != torch.Tensor: print("Converted tensor") output_std = torch.from_numpy(output_std) if type(output) != torch.Tensor: output = torch.from_numpy(output) output_dist = torch.distributions.Normal(output, output_std) if validation_dataset: src, output, labels, output_dist = handle_scaling( validation_dataset, src, output, labels, probabilistic, m, output_std) if probabilistic: loss = -output_dist.log_prob(labels.float()).sum() # FIX THIS? elif isinstance(criterion, MASELoss): assert len(labels.shape) == len(output.shape) loss = criterion(labels.float(), output, src, m) else: assert len(labels.shape) == len(output.shape) assert labels.shape[0] == output.shape[0] loss = criterion(output, labels.float()) return loss
def evaluate_model( model: Type[TimeSeriesModel], model_type: str, target_col: List[str], evaluation_metrics: List, inference_params: Dict, eval_log: Dict, ) -> Tuple[Dict, pd.DataFrame, int, pd.DataFrame]: """ A function to evaluate a model. Called automatically at end of training. Can be imported for continuing to evaluate a model in other places as well. .. highlight:: python .. code-block:: python from flood_forecast.evaluator import evaluate_model forecast_model = PyTorchForecast() evaluate_model(forecast_model, ) ... ''' """ if model_type == "PyTorch": ( df_train_and_test, end_tensor, forecast_history, forecast_start_idx, test_data, df_predictions, # df_prediction_samples_std_dev, ) = infer_on_torch_model(model, **inference_params) # To-do turn this into a general function g_loss = False probablistic = True if "probabilistic" in inference_params else False if isinstance(end_tensor, tuple) and not probablistic: end_tensor_0 = end_tensor[1] end_tensor = end_tensor[0] g_loss = True print("transform end tens preform") if test_data.scale: print("Un-transforming data") if probablistic: print('probabilistic running on infer_on_torch_model') end_tensor_mean = test_data.inverse_scale(end_tensor[0].detach().reshape(-1, 1)) end_tensor_list = flatten_list_function(end_tensor_mean.numpy().tolist()) end_tensor_mean = end_tensor_mean.squeeze(1) else: if "n_targets" in model.params: end_tensor = test_data.inverse_scale(end_tensor.detach()) else: end_tensor = test_data.inverse_scale(end_tensor.detach().reshape(-1, 1)) end_tensor_list = flatten_list_function(end_tensor.numpy().tolist()) end_tensor = end_tensor.squeeze(1) # Removing extra dim from reshape? history_length = model.params["dataset_params"]["forecast_history"] if "n_targets" in model.params: df_train_and_test.loc[df_train_and_test.index[history_length:], "preds"] = end_tensor[:, 0].numpy().tolist() for i, target in enumerate(target_col): df_train_and_test["pred_" + target] = 0 df_train_and_test.loc[df_train_and_test.index[history_length:], "pred_" + target] = end_tensor[:, i].numpy().tolist() else: df_train_and_test.loc[df_train_and_test.index[history_length:], "preds"] = end_tensor_list df_train_and_test["pred_" + target_col[0]] = 0 df_train_and_test.loc[df_train_and_test.index[history_length:], "pred_" + target_col[0]] = end_tensor_list print("Current historical dataframe ") print(df_train_and_test) for evaluation_metric in model.crit: idx = 0 for target in target_col: labels = torch.from_numpy(df_train_and_test[target][forecast_history:].to_numpy()) evaluation_metric_function = evaluation_metric if "probabilistic" in inference_params: s = evaluation_metric_function( torch.distributions.Normal(end_tensor[0], end_tensor[1][0]), labels, ) elif isinstance(evaluation_metric_function, MASELoss): s = evaluation_metric_function( labels, end_tensor, torch.from_numpy( df_train_and_test[target][:forecast_history].to_numpy() ) ) elif g_loss: g = GaussianLoss(end_tensor.unsqueeze(1), end_tensor_0.unsqueeze(1)) s = g(labels.unsqueeze(1)) else: if "n_targets" in model.params: s = evaluation_metric_function( labels, end_tensor[:, idx], ) else: s = evaluation_metric_function( labels, end_tensor, ) idx += 1 eval_log[target + "_" + evaluation_metric.__class__.__name__] = s # Explain model behaviour using shap if "probabilistic" in inference_params: print("Probabilistic explainability currently not supported.") elif "n_targets" in model.params: print("Multitask forecasting support coming soon") elif g_loss: print("SHAP not yet supported for these models with multiple outputs") else: deep_explain_model_summary_plot( model, test_data, inference_params["datetime_start"] ) deep_explain_model_heatmap(model, test_data, inference_params["datetime_start"]) return eval_log, df_train_and_test, forecast_start_idx, df_predictions
def compute_validation(validation_loader: DataLoader, # s lint model, epoch: int, sequence_size: int, criterion: Type[torch.nn.modules.loss._Loss], device: torch.device, decoder_structure=False, meta_data_model=None, use_wandb: bool = False, meta_model=None, val_or_test="validation_loss", probabilistic=False) -> float: """ Function to compute the validation or the test loss """ print('compute_validation') model.eval() loop_loss = 0.0 with torch.no_grad(): i = 0 loss_unscaled_full = 0.0 for src, targ in validation_loader: src = src.to(device) targ = targ.to(device) i += 1 if decoder_structure: if type(model).__name__ == "SimpleTransformer": targ_clone = targ.detach().clone() output = greedy_decode( model, src, targ.shape[1], targ_clone, device=device)[ :, :, 0] else: if probabilistic: output, output_std = simple_decode(model, src, targ.shape[1], targ, 1, probabilistic=probabilistic) output, output_std = output[:, :, 0], output_std[0] output_dist = torch.distributions.Normal(output, output_std) else: output = simple_decode(model=model, src=src, max_seq_len=targ.shape[1], real_target=targ, output_len=1, probabilistic=probabilistic)[:, :, 0] else: if probabilistic: output_dist = model(src.float()) output = output_dist.mean.detach().numpy() output_std = output_dist.stddev.detach().numpy() else: output = model(src.float()) labels = targ[:, :, 0] validation_dataset = validation_loader.dataset if validation_dataset.scale: unscaled_labels = validation_dataset.inverse_scale(labels) if probabilistic: unscaled_out = validation_dataset.inverse_scale(output) try: output_std = numpy_to_tvar(output_std) except Exception: pass unscaled_dist = torch.distributions.Normal(unscaled_out, output_std) loss_unscaled = -unscaled_dist.log_prob(unscaled_labels.float()).sum() # FIX THIS loss_unscaled_full += len(labels.float()) * loss_unscaled.numpy().item() else: # unscaled_src = validation_dataset.scale.inverse_transform(src.cpu()) unscaled_out = validation_dataset.inverse_scale(output.cpu()) unscaled_labels = validation_dataset.inverse_scale(labels.cpu()) loss_unscaled = criterion(unscaled_out, unscaled_labels.float()) loss_unscaled_full += len(labels.float()) * loss_unscaled.item() if i % 10 == 0 and use_wandb: wandb.log({"trg": unscaled_labels, "model_pred": unscaled_out}) if probabilistic: loss = -output_dist.log_prob(labels.float()).sum() # FIX THIS loss = loss.numpy() elif isinstance(criterion, GaussianLoss): g_loss = GaussianLoss(output[0], output[1]) loss = g_loss(labels) else: loss = criterion(output, labels.float()) loop_loss += len(labels.float()) * loss.item() if use_wandb: if loss_unscaled_full: tot_unscaled_loss = loss_unscaled_full / (len(validation_loader.dataset) - 1) wandb.log({'epoch': epoch, val_or_test: loop_loss / (len(validation_loader.dataset) - 1), "unscaled_" + val_or_test: tot_unscaled_loss}) else: wandb.log({'epoch': epoch, val_or_test: loop_loss / (len(validation_loader.dataset) - 1)}) model.train() return loop_loss / (len(validation_loader.dataset) - 1)
def compute_loss(labels, output, src, criterion, validation_dataset, probabilistic=None, output_std=None, m=1): """Function for computing the loss :param labels: The real forecasted values :type labels: torch.Tensor :param output: The output of the model :type output: torch.Tensor :param src: The source values (only really needed for MASELoss) :type src: torch.Tensor :param criterion: [description] :type criterion: [type] :param validation_dataset: [description] :type validation_dataset: [type] :param probabilistic: [description], defaults to None :type probabilistic: [type], optional :param output_std: [description], defaults to None :type output_std: [type], optional :param m: [description], defaults to 1 :type m: int, optional :return: [description] :rtype: [type] """ if not probabilistic and isinstance(output, torch.Tensor): if len(labels.shape) != len(output.shape): if len(labels.shape) > 1: if labels.shape[1] == output.shape[1]: labels = labels.unsqueeze(2) else: labels = labels.unsqueeze(0) if probabilistic: if type(output_std) != torch.Tensor: print("Converted") output_std = torch.from_numpy(output_std) if type(output) != torch.Tensor: output = torch.from_numpy(output) output_dist = torch.distributions.Normal(output, output_std) if validation_dataset: if probabilistic: unscaled_out = validation_dataset.inverse_scale(output) try: output_std = numpy_to_tvar(output_std) except Exception: pass output_dist = torch.distributions.Normal(unscaled_out, output_std) elif m > 1: output = validation_dataset.inverse_scale(output.cpu()) labels = validation_dataset.inverse_scale(labels.cpu()) elif len(output.shape) == 3: output = output.cpu().numpy().transpose(0, 2, 1) labels = labels.cpu().numpy().transpose(0, 2, 1) output = validation_dataset.inverse_scale(torch.from_numpy(output)) labels = validation_dataset.inverse_scale(torch.from_numpy(labels)) stuff = src.cpu().numpy().transpose(0, 2, 1) src = validation_dataset.inverse_scale(torch.from_numpy(stuff)) else: output = validation_dataset.inverse_scale(output.cpu().transpose(1, 0)) labels = validation_dataset.inverse_scale(labels.cpu().transpose(1, 0)) src = validation_dataset.inverse_scale(src.cpu().transpose(1, 0)) if probabilistic: loss = -output_dist.log_prob(labels.float()).sum() # FIX THIS elif isinstance(criterion, GaussianLoss): g_loss = GaussianLoss(output[0], output[1]) loss = g_loss(labels) elif isinstance(criterion, MASELoss): assert len(labels.shape) == len(output.shape) loss = criterion(labels.float(), output, src, m) else: assert len(labels.shape) == len(output.shape) assert labels.shape[0] == output.shape[0] loss = criterion(output, labels.float()) return loss