def make_plots(self, date: datetime, csv_path: str = None, csv_bucket: str = None, save_name=None, wandb_plot_id=None): """ :param date: [description] :type date: datetime :param csv_path: [description], defaults to None :type csv_path: str, optional :param csv_bucket: [description], defaults to None :type csv_bucket: str, optional :param save_name: [description], defaults to None :type save_name: [type], optional :param wandb_plot_id: [description], defaults to None :type wandb_plot_id: [type], optional :return: [description] :rtype: [type] """ if csv_path is None: csv_path = self.csv_path df, tensor, history, forecast_start, test, samples = self.infer_now( date, csv_path, csv_bucket, save_name) plt = {} for sample in samples: plt = plot_df_test_with_confidence_interval( df, sample, forecast_start, self.model.params) if wandb_plot_id: wandb.log({wandb_plot_id: plt}) deep_explain_model_summary_plot(self.model, test, date) deep_explain_model_heatmap(self.model, test, date) return tensor, history, test, plt
def make_plots(self, date: datetime, csv_path: str = None, csv_bucket: str = None, save_name=None, wandb_plot_id=None): """Function to create plots in inference mode. :param date: The datetime to start inference :type date: datetime :param csv_path: The path to the CSV file you want to use for inference, defaults to None :type csv_path: str, optional :param csv_bucket: [description], defaults to None :type csv_bucket: str, optional :param save_name: [description], defaults to None :type save_name: [type], optional :param wandb_plot_id: [description], defaults to None :type wandb_plot_id: [type], optional :return: [description] :rtype: tuple(torch.Tensor, torch.Tensor, CSVTestLoader, matplotlib.pyplot.plot) """ if csv_path is None: csv_path = self.csv_path df, tensor, history, forecast_start, test, samples = self.infer_now(date, csv_path, csv_bucket, save_name) plt = {} for sample, targ in zip(samples, self.model.params["dataset_params"]["target_col"]): plt = plot_df_test_with_confidence_interval(df, sample, forecast_start, self.model.params, targ) if wandb_plot_id: wandb.log({wandb_plot_id + targ: plt}) if not self.n_targets: deep_explain_model_summary_plot(self.model, test, date) deep_explain_model_heatmap(self.model, test, date) return tensor, history, test, plt
def evaluate_model( model: Type[TimeSeriesModel], model_type: str, target_col: List[str], evaluation_metrics: List, inference_params: Dict, eval_log: Dict, ) -> Tuple[Dict, pd.DataFrame, int, pd.DataFrame]: """ A function to evaluate a model. Requires a model of type TimeSeriesModel """ if model_type == "PyTorch": ( df_train_and_test, end_tensor, forecast_history, forecast_start_idx, test_data, df_predictions, ) = infer_on_torch_model(model, **inference_params) # Unscale test data if scaler was applied print("test_data scale") if test_data.scale: print("Un-transforming data") end_tensor = test_data.inverse_scale(end_tensor.detach().reshape( -1, 1)) end_tensor_list = flatten_list_function( end_tensor.numpy().tolist()) history_length = model.params["dataset_params"]["forecast_history"] df_train_and_test["preds"][history_length:] = end_tensor_list end_tensor = end_tensor.squeeze( 1) # Removing extra dim from reshape? df_predictions = pd.DataFrame( test_data.inverse_scale(df_predictions).numpy(), index=df_predictions.index, ) print("Current historical dataframe") print(df_train_and_test) for evaluation_metric in evaluation_metrics: for target in target_col: eval_params = {} evaluation_metric_function = pytorch_criterion_dict[ evaluation_metric](**eval_params) s = evaluation_metric_function( torch.from_numpy( df_train_and_test[target][forecast_history:].to_numpy()), end_tensor, ) eval_log[target + "_" + evaluation_metric] = s # Explain model behaviour using shap deep_explain_model_summary_plot(model, test_data, inference_params["datetime_start"]) deep_explain_model_heatmap(model, test_data, inference_params["datetime_start"]) return eval_log, df_train_and_test, forecast_start_idx, df_predictions
def test_deep_explain_model_summary_plot(self): deep_explain_model_summary_plot( model=self.model, datetime_start=datetime.datetime(2014, 6, 2, 0), test_csv_path=os.path.join(self.test_path2, "keag_small.csv"), dataset_params=self.data_base_params, ) # dummy assert self.assertEqual(1, 1)
def make_plots(self, date: datetime, csv_path: str = None, csv_bucket: str = None, save_name=None, wandb_plot_id=None): if csv_path is None: csv_path = self.csv_path df, tensor, history, forecast_start, test, samples = self.infer_now( date, csv_path, csv_bucket, save_name) plt = plot_df_test_with_confidence_interval(df, samples, forecast_start, self.model.params) if wandb_plot_id: wandb.log({wandb_plot_id: plt}) deep_explain_model_summary_plot(self.model, test, date) deep_explain_model_heatmap(self.model, test, date) return tensor, history, test, plt
def test_deep_explain_model_summary_plot(self): deep_explain_model_summary_plot(model=self.model, csv_test_loader=self.csv_test_loader) deep_explain_model_summary_plot(model=self.lstm_model, csv_test_loader=self.csv_test_loader) deep_explain_model_summary_plot(model=self.simple_linear_model, csv_test_loader=self.csv_test_loader) # dummy assert self.assertEqual(1, 1)
def evaluate_model( model: Type[TimeSeriesModel], model_type: str, target_col: List[str], evaluation_metrics: List, inference_params: Dict, eval_log: Dict, ) -> Tuple[Dict, pd.DataFrame, int, pd.DataFrame]: """ A function to evaluate a model. Called automatically at end of training. Can be imported for continuing to evaluate a model in other places as well. .. highlight:: python .. code-block:: python from flood_forecast.evaluator import evaluate_model evaluate_model(model, ) ... ''' """ if model_type == "PyTorch": ( df_train_and_test, end_tensor, forecast_history, forecast_start_idx, test_data, df_predictions, # df_prediction_samples_std_dev, ) = infer_on_torch_model(model, **inference_params) # To-do turn this into a general function if test_data.scale: print("Un-transforming data") if "probabilistic" in inference_params: print('probabilistic in infer_on_torch_model') end_tensor_mean = test_data.inverse_scale( end_tensor[0].detach().reshape(-1, 1)) end_tensor_list = flatten_list_function( end_tensor_mean.numpy().tolist()) end_tensor_mean = end_tensor_mean.squeeze(1) else: print(end_tensor.shape) if "n_targets" in model.params: end_tensor = test_data.inverse_scale(end_tensor.detach()) else: end_tensor = test_data.inverse_scale( end_tensor.detach().reshape(-1, 1)) end_tensor_list = flatten_list_function( end_tensor.numpy().tolist()) end_tensor = end_tensor.squeeze( 1) # Removing extra dim from reshape? history_length = model.params["dataset_params"]["forecast_history"] if "n_targets" in model.params: df_train_and_test["preds"][ history_length:] = end_tensor[:, 0].numpy().tolist() for i, target in enumerate(target_col): df_train_and_test["pred_" + target] = 0 df_train_and_test["pred_" + target][ history_length:] = end_tensor[:, i].numpy().tolist() else: df_train_and_test["preds"][history_length:] = end_tensor_list df_train_and_test["pred_" + target_col[0]] = 0 df_train_and_test[ "pred_" + target_col[0]][history_length:] = end_tensor_list print("Current historical dataframe ") print(df_train_and_test) for evaluation_metric in model.crit: idx = 0 for target in target_col: evaluation_metric_function = evaluation_metric if "probabilistic" in inference_params: s = evaluation_metric_function( torch.distributions.Normal(end_tensor[0], end_tensor[1][0]), torch.from_numpy(df_train_and_test[target] [forecast_history:].to_numpy()), ) elif isinstance(evaluation_metric_function, MASELoss): s = evaluation_metric_function( torch.from_numpy(df_train_and_test[target] [forecast_history:].to_numpy()), end_tensor, torch.from_numpy(df_train_and_test[target] [:forecast_history].to_numpy())) else: if "n_targets" in model.params: s = evaluation_metric_function( torch.from_numpy(df_train_and_test[target] [forecast_history:].to_numpy()), end_tensor[:, idx], ) else: s = evaluation_metric_function( torch.from_numpy(df_train_and_test[target] [forecast_history:].to_numpy()), end_tensor, ) idx += 1 eval_log[target + "_" + evaluation_metric.__class__.__name__] = s # Explain model behaviour using shap if "probabilistic" in inference_params: print("Probabilistic explainability currently not supported.") elif "n_targets" in model.params: print("Multitask forecasting support coming soon") else: deep_explain_model_summary_plot(model, test_data, inference_params["datetime_start"]) deep_explain_model_heatmap(model, test_data, inference_params["datetime_start"]) return eval_log, df_train_and_test, forecast_start_idx, df_predictions
def evaluate_model( model: Type[TimeSeriesModel], model_type: str, target_col: List[str], evaluation_metrics: List, inference_params: Dict, eval_log: Dict, ) -> Tuple[Dict, pd.DataFrame, int, pd.DataFrame]: """ A function to evaluate a model. Requires a model of type TimeSeriesModel """ if model_type == "PyTorch": ( df_train_and_test, end_tensor, forecast_history, forecast_start_idx, test_data, df_predictions, # df_prediction_samples_std_dev, ) = infer_on_torch_model(model, **inference_params) # To-do turn this into a general function print("test_data scale") if test_data.scale: print("Un-transforming data") if "probabilistic" in inference_params: print('probabilistic in infer_on_torch_model') end_tensor_mean = test_data.inverse_scale( end_tensor[0].detach().reshape(-1, 1)) end_tensor_list = flatten_list_function( end_tensor_mean.numpy().tolist()) end_tensor_mean = end_tensor_mean.squeeze(1) else: print(end_tensor.shape) if "n_targets" in model.params: end_tensor = test_data.inverse_scale(end_tensor.detach()) else: end_tensor = test_data.inverse_scale( end_tensor.detach().reshape(-1, 1)) end_tensor_list = flatten_list_function( end_tensor.numpy().tolist()) end_tensor = end_tensor.squeeze( 1) # Removing extra dim from reshape? history_length = model.params["dataset_params"]["forecast_history"] if "n_targets" in model.params: df_train_and_test["preds"][ history_length:] = end_tensor[:, 0].numpy().tolist() for i in range(1, model.params["n_targets"]): df_train_and_test["pred_" + str(i)] = end_tensor[:, i].numpy().tolist else: df_train_and_test["preds"][history_length:] = end_tensor_list print('end_tensor', end_tensor) if len(df_predictions.columns > 0): df_predictions = pd.DataFrame( test_data.inverse_scale(df_predictions).numpy(), index=df_predictions.index, ) print("Current historical dataframe ") print(df_train_and_test) for evaluation_metric in model.crit: idx = 0 for target in target_col: evaluation_metric_function = evaluation_metric if "probabilistic" in inference_params: s = evaluation_metric_function( torch.distributions.Normal(end_tensor[0], end_tensor[1][0]), torch.from_numpy(df_train_and_test[target] [forecast_history:].to_numpy()), ) elif isinstance(evaluation_metric_function, MASELoss): s = evaluation_metric_function( torch.from_numpy(df_train_and_test[target] [forecast_history:].to_numpy()), end_tensor, torch.from_numpy(df_train_and_test[target] [:forecast_history].to_numpy())) else: if "n_targets" in model.params: s = evaluation_metric_function( torch.from_numpy(df_train_and_test[target] [forecast_history:].to_numpy()), end_tensor[:, idx], ) else: s = evaluation_metric_function( torch.from_numpy(df_train_and_test[target] [forecast_history:].to_numpy()), end_tensor, ) idx += 1 eval_log[target + "_" + evaluation_metric.__class__.__name__] = s # Explain model behaviour using shap if "probabilistic" in inference_params: print("Probabilistic explainability currently not supported.") elif "n_targets" in model.params: print("Multitask forecasting support coming soon") else: deep_explain_model_summary_plot(model, test_data, inference_params["datetime_start"]) deep_explain_model_heatmap(model, test_data, inference_params["datetime_start"]) return eval_log, df_train_and_test, forecast_start_idx, df_predictions
def evaluate_model( model: Type[TimeSeriesModel], model_type: str, target_col: List[str], evaluation_metrics: List, inference_params: Dict, eval_log: Dict, ) -> Tuple[Dict, pd.DataFrame, int, pd.DataFrame]: """ A function to evaluate a model. Called automatically at end of training. Can be imported for continuing to evaluate a model in other places as well. .. highlight:: python .. code-block:: python from flood_forecast.evaluator import evaluate_model forecast_model = PyTorchForecast(config_file) e_log, df_train_test, f_idx, df_preds = evaluate_model(forecast_model, "PyTorch", ["cfs"], ["MSE", "MAPE"], {}) print(e_log) # {"MSE":0.2, "MAPE":0.1} print(df_train_test) # will print a pandas dataframe ... ''' """ if model_type == "PyTorch": ( df_train_and_test, end_tensor, forecast_history, forecast_start_idx, test_data, df_predictions, # df_prediction_samples_std_dev, ) = infer_on_torch_model(model, **inference_params) # To-do turn this into a general function g_loss = False probablistic = True if "probabilistic" in inference_params else False if isinstance(end_tensor, tuple) and not probablistic: end_tensor_0 = end_tensor[1] end_tensor = end_tensor[0] g_loss = True if test_data.scale: print("Un-transforming data") if probablistic: print('probabilistic running on infer_on_torch_model') end_tensor_mean = test_data.inverse_scale( end_tensor[0].detach().reshape(-1, 1)) end_tensor_list = flatten_list_function( end_tensor_mean.numpy().tolist()) end_tensor_mean = end_tensor_mean.squeeze(1) else: if "n_targets" in model.params: if model.params["model_name"] == "Informer": end_tensor = end_tensor[:, :, 0:model.params["n_targets"]] end_tensor = test_data.inverse_scale(end_tensor.detach()) else: end_tensor = test_data.inverse_scale( end_tensor.detach().reshape(-1, 1)) end_tensor_list = flatten_list_function( end_tensor.numpy().tolist()) end_tensor = end_tensor.squeeze( 1) # Removing extra dim from reshape? history_length = model.params["dataset_params"]["forecast_history"] if "n_targets" in model.params: df_train_and_test.loc[ df_train_and_test.index[history_length:], "preds"] = end_tensor[:, 0].numpy().tolist() for i, target in enumerate(target_col): df_train_and_test["pred_" + target] = 0 df_train_and_test.loc[ df_train_and_test.index[history_length:], "pred_" + target] = end_tensor[:, i].numpy().tolist() else: df_train_and_test.loc[df_train_and_test.index[history_length:], "preds"] = end_tensor_list df_train_and_test["pred_" + target_col[0]] = 0 df_train_and_test.loc[df_train_and_test.index[history_length:], "pred_" + target_col[0]] = end_tensor_list print("Current historical dataframe ") print(df_train_and_test) for evaluation_metric in model.crit: idx = 0 for target in target_col: labels = torch.from_numpy( df_train_and_test[target][forecast_history:].to_numpy()) evaluation_metric_function = evaluation_metric if "probabilistic" in inference_params: s = evaluation_metric_function( torch.distributions.Normal(end_tensor[0], end_tensor[1][0]), labels, ) elif isinstance(evaluation_metric_function, MASELoss): s = evaluation_metric_function( labels, end_tensor, torch.from_numpy(df_train_and_test[target] [:forecast_history].to_numpy())) elif g_loss: g = GaussianLoss(end_tensor.unsqueeze(1), end_tensor_0.unsqueeze(1)) s = g(labels.unsqueeze(1)) else: if "n_targets" in model.params: s = evaluation_metric_function( labels, end_tensor[:, idx], ) else: s = evaluation_metric_function( labels, end_tensor, ) idx += 1 eval_log[target + "_" + evaluation_metric.__class__.__name__] = s # Explain model behaviour using shap if "probabilistic" in inference_params: print("Probabilistic explainability currently not supported.") elif "n_targets" in model.params: print("Multitask forecasting support coming soon") elif g_loss: print("SHAP not yet supported for these models with multiple outputs") else: deep_explain_model_summary_plot(model, test_data, inference_params["datetime_start"]) deep_explain_model_heatmap(model, test_data, inference_params["datetime_start"]) return eval_log, df_train_and_test, forecast_start_idx, df_predictions