def make_plots(self, date: datetime, csv_path: str = None, csv_bucket: str = None,
save_name=None, wandb_plot_id=None):
"""Function to create plots in inference mode.
:param date: The datetime to start inference
:type date: datetime
:param csv_path: The path to the CSV file you want to use for inference, defaults to None
:type csv_path: str, optional
:param csv_bucket: [description], defaults to None
:type csv_bucket: str, optional
:param save_name: [description], defaults to None
:type save_name: [type], optional
:param wandb_plot_id: [description], defaults to None
:type wandb_plot_id: [type], optional
:return: [description]
:rtype: tuple(torch.Tensor, torch.Tensor, CSVTestLoader, matplotlib.pyplot.plot)
"""
if csv_path is None:
csv_path = self.csv_path
df, tensor, history, forecast_start, test, samples = self.infer_now(date, csv_path, csv_bucket, save_name)
plt = {}
for sample, targ in zip(samples, self.model.params["dataset_params"]["target_col"]):
plt = plot_df_test_with_confidence_interval(df, sample, forecast_start, self.model.params, targ)
if wandb_plot_id:
wandb.log({wandb_plot_id + targ: plt})
if not self.n_targets:
deep_explain_model_summary_plot(self.model, test, date)
deep_explain_model_heatmap(self.model, test, date)
return tensor, history, test, plt
def make_plots(self,
date: datetime,
csv_path: str = None,
csv_bucket: str = None,
save_name=None,
wandb_plot_id=None):
"""
:param date: [description]
:type date: datetime
:param csv_path: [description], defaults to None
:type csv_path: str, optional
:param csv_bucket: [description], defaults to None
:type csv_bucket: str, optional
:param save_name: [description], defaults to None
:type save_name: [type], optional
:param wandb_plot_id: [description], defaults to None
:type wandb_plot_id: [type], optional
:return: [description]
:rtype: [type]
"""
if csv_path is None:
csv_path = self.csv_path
df, tensor, history, forecast_start, test, samples = self.infer_now(
date, csv_path, csv_bucket, save_name)
plt = {}
for sample in samples:
plt = plot_df_test_with_confidence_interval(
df, sample, forecast_start, self.model.params)
if wandb_plot_id:
wandb.log({wandb_plot_id: plt})
deep_explain_model_summary_plot(self.model, test, date)
deep_explain_model_heatmap(self.model, test, date)
return tensor, history, test, plt
def evaluate_model(
model: Type[TimeSeriesModel],
model_type: str,
target_col: List[str],
evaluation_metrics: List,
inference_params: Dict,
eval_log: Dict,
) -> Tuple[Dict, pd.DataFrame, int, pd.DataFrame]:
"""
A function to evaluate a model.
Requires a model of type TimeSeriesModel
"""
if model_type == "PyTorch":
(
df_train_and_test,
end_tensor,
forecast_history,
forecast_start_idx,
test_data,
df_predictions,
) = infer_on_torch_model(model, **inference_params)
# Unscale test data if scaler was applied
print("test_data scale")
if test_data.scale:
print("Un-transforming data")
end_tensor = test_data.inverse_scale(end_tensor.detach().reshape(
-1, 1))
end_tensor_list = flatten_list_function(
end_tensor.numpy().tolist())
history_length = model.params["dataset_params"]["forecast_history"]
df_train_and_test["preds"][history_length:] = end_tensor_list
end_tensor = end_tensor.squeeze(
1) # Removing extra dim from reshape?
df_predictions = pd.DataFrame(
test_data.inverse_scale(df_predictions).numpy(),
index=df_predictions.index,
)
print("Current historical dataframe")
print(df_train_and_test)
for evaluation_metric in evaluation_metrics:
for target in target_col:
eval_params = {}
evaluation_metric_function = pytorch_criterion_dict[
evaluation_metric](**eval_params)
s = evaluation_metric_function(
torch.from_numpy(
df_train_and_test[target][forecast_history:].to_numpy()),
end_tensor,
)
eval_log[target + "_" + evaluation_metric] = s
# Explain model behaviour using shap
deep_explain_model_summary_plot(model, test_data,
inference_params["datetime_start"])
deep_explain_model_heatmap(model, test_data,
inference_params["datetime_start"])
return eval_log, df_train_and_test, forecast_start_idx, df_predictions
def make_plots(self,
date: datetime,
csv_path: str = None,
csv_bucket: str = None,
save_name=None,
wandb_plot_id=None):
if csv_path is None:
csv_path = self.csv_path
df, tensor, history, forecast_start, test, samples = self.infer_now(
date, csv_path, csv_bucket, save_name)
plt = plot_df_test_with_confidence_interval(df, samples,
forecast_start,
self.model.params)
if wandb_plot_id:
wandb.log({wandb_plot_id: plt})
deep_explain_model_summary_plot(self.model, test, date)
deep_explain_model_heatmap(self.model, test, date)
return tensor, history, test, plt
def test_deep_explain_model_heatmap(self):
deep_explain_model_heatmap(model=self.model,
csv_test_loader=self.csv_test_loader)
deep_explain_model_heatmap(model=self.lstm_model,
csv_test_loader=self.csv_test_loader)
deep_explain_model_heatmap(model=self.simple_linear_model,
csv_test_loader=self.csv_test_loader)
# dummy assert
self.assertEqual(1, 1)
def evaluate_model(
model: Type[TimeSeriesModel],
model_type: str,
target_col: List[str],
evaluation_metrics: List,
inference_params: Dict,
eval_log: Dict,
) -> Tuple[Dict, pd.DataFrame, int, pd.DataFrame]:
"""
A function to evaluate a model. Called automatically at end of training.
Can be imported for continuing to evaluate a model in other places as well.
.. highlight:: python
.. code-block:: python
from flood_forecast.evaluator import evaluate_model
evaluate_model(model, )
...
'''
"""
if model_type == "PyTorch":
(
df_train_and_test,
end_tensor,
forecast_history,
forecast_start_idx,
test_data,
df_predictions,
# df_prediction_samples_std_dev,
) = infer_on_torch_model(model, **inference_params)
# To-do turn this into a general function
if test_data.scale:
print("Un-transforming data")
if "probabilistic" in inference_params:
print('probabilistic in infer_on_torch_model')
end_tensor_mean = test_data.inverse_scale(
end_tensor[0].detach().reshape(-1, 1))
end_tensor_list = flatten_list_function(
end_tensor_mean.numpy().tolist())
end_tensor_mean = end_tensor_mean.squeeze(1)
else:
print(end_tensor.shape)
if "n_targets" in model.params:
end_tensor = test_data.inverse_scale(end_tensor.detach())
else:
end_tensor = test_data.inverse_scale(
end_tensor.detach().reshape(-1, 1))
end_tensor_list = flatten_list_function(
end_tensor.numpy().tolist())
end_tensor = end_tensor.squeeze(
1) # Removing extra dim from reshape?
history_length = model.params["dataset_params"]["forecast_history"]
if "n_targets" in model.params:
df_train_and_test["preds"][
history_length:] = end_tensor[:, 0].numpy().tolist()
for i, target in enumerate(target_col):
df_train_and_test["pred_" + target] = 0
df_train_and_test["pred_" + target][
history_length:] = end_tensor[:, i].numpy().tolist()
else:
df_train_and_test["preds"][history_length:] = end_tensor_list
df_train_and_test["pred_" + target_col[0]] = 0
df_train_and_test[
"pred_" + target_col[0]][history_length:] = end_tensor_list
print("Current historical dataframe ")
print(df_train_and_test)
for evaluation_metric in model.crit:
idx = 0
for target in target_col:
evaluation_metric_function = evaluation_metric
if "probabilistic" in inference_params:
s = evaluation_metric_function(
torch.distributions.Normal(end_tensor[0],
end_tensor[1][0]),
torch.from_numpy(df_train_and_test[target]
[forecast_history:].to_numpy()),
)
elif isinstance(evaluation_metric_function, MASELoss):
s = evaluation_metric_function(
torch.from_numpy(df_train_and_test[target]
[forecast_history:].to_numpy()),
end_tensor,
torch.from_numpy(df_train_and_test[target]
[:forecast_history].to_numpy()))
else:
if "n_targets" in model.params:
s = evaluation_metric_function(
torch.from_numpy(df_train_and_test[target]
[forecast_history:].to_numpy()),
end_tensor[:, idx],
)
else:
s = evaluation_metric_function(
torch.from_numpy(df_train_and_test[target]
[forecast_history:].to_numpy()),
end_tensor,
)
idx += 1
eval_log[target + "_" + evaluation_metric.__class__.__name__] = s
# Explain model behaviour using shap
if "probabilistic" in inference_params:
print("Probabilistic explainability currently not supported.")
elif "n_targets" in model.params:
print("Multitask forecasting support coming soon")
else:
deep_explain_model_summary_plot(model, test_data,
inference_params["datetime_start"])
deep_explain_model_heatmap(model, test_data,
inference_params["datetime_start"])
return eval_log, df_train_and_test, forecast_start_idx, df_predictions
def evaluate_model(
model: Type[TimeSeriesModel],
model_type: str,
target_col: List[str],
evaluation_metrics: List,
inference_params: Dict,
eval_log: Dict,
) -> Tuple[Dict, pd.DataFrame, int, pd.DataFrame]:
"""
A function to evaluate a model.
Requires a model of type TimeSeriesModel
"""
if model_type == "PyTorch":
(
df_train_and_test,
end_tensor,
forecast_history,
forecast_start_idx,
test_data,
df_predictions,
# df_prediction_samples_std_dev,
) = infer_on_torch_model(model, **inference_params)
# To-do turn this into a general function
print("test_data scale")
if test_data.scale:
print("Un-transforming data")
if "probabilistic" in inference_params:
print('probabilistic in infer_on_torch_model')
end_tensor_mean = test_data.inverse_scale(
end_tensor[0].detach().reshape(-1, 1))
end_tensor_list = flatten_list_function(
end_tensor_mean.numpy().tolist())
end_tensor_mean = end_tensor_mean.squeeze(1)
else:
print(end_tensor.shape)
if "n_targets" in model.params:
end_tensor = test_data.inverse_scale(end_tensor.detach())
else:
end_tensor = test_data.inverse_scale(
end_tensor.detach().reshape(-1, 1))
end_tensor_list = flatten_list_function(
end_tensor.numpy().tolist())
end_tensor = end_tensor.squeeze(
1) # Removing extra dim from reshape?
history_length = model.params["dataset_params"]["forecast_history"]
if "n_targets" in model.params:
df_train_and_test["preds"][
history_length:] = end_tensor[:, 0].numpy().tolist()
for i in range(1, model.params["n_targets"]):
df_train_and_test["pred_" +
str(i)] = end_tensor[:, i].numpy().tolist
else:
df_train_and_test["preds"][history_length:] = end_tensor_list
print('end_tensor', end_tensor)
if len(df_predictions.columns > 0):
df_predictions = pd.DataFrame(
test_data.inverse_scale(df_predictions).numpy(),
index=df_predictions.index,
)
print("Current historical dataframe ")
print(df_train_and_test)
for evaluation_metric in model.crit:
idx = 0
for target in target_col:
evaluation_metric_function = evaluation_metric
if "probabilistic" in inference_params:
s = evaluation_metric_function(
torch.distributions.Normal(end_tensor[0],
end_tensor[1][0]),
torch.from_numpy(df_train_and_test[target]
[forecast_history:].to_numpy()),
)
elif isinstance(evaluation_metric_function, MASELoss):
s = evaluation_metric_function(
torch.from_numpy(df_train_and_test[target]
[forecast_history:].to_numpy()),
end_tensor,
torch.from_numpy(df_train_and_test[target]
[:forecast_history].to_numpy()))
else:
if "n_targets" in model.params:
s = evaluation_metric_function(
torch.from_numpy(df_train_and_test[target]
[forecast_history:].to_numpy()),
end_tensor[:, idx],
)
else:
s = evaluation_metric_function(
torch.from_numpy(df_train_and_test[target]
[forecast_history:].to_numpy()),
end_tensor,
)
idx += 1
eval_log[target + "_" + evaluation_metric.__class__.__name__] = s
# Explain model behaviour using shap
if "probabilistic" in inference_params:
print("Probabilistic explainability currently not supported.")
elif "n_targets" in model.params:
print("Multitask forecasting support coming soon")
else:
deep_explain_model_summary_plot(model, test_data,
inference_params["datetime_start"])
deep_explain_model_heatmap(model, test_data,
inference_params["datetime_start"])
return eval_log, df_train_and_test, forecast_start_idx, df_predictions
def evaluate_model(
model: Type[TimeSeriesModel],
model_type: str,
target_col: List[str],
evaluation_metrics: List,
inference_params: Dict,
eval_log: Dict,
) -> Tuple[Dict, pd.DataFrame, int, pd.DataFrame]:
"""
A function to evaluate a model. Called automatically at end of training.
Can be imported for continuing to evaluate a model in other places as well.
.. highlight:: python
.. code-block:: python
from flood_forecast.evaluator import evaluate_model
forecast_model = PyTorchForecast(config_file)
e_log, df_train_test, f_idx, df_preds = evaluate_model(forecast_model, "PyTorch", ["cfs"], ["MSE", "MAPE"], {})
print(e_log) # {"MSE":0.2, "MAPE":0.1}
print(df_train_test) # will print a pandas dataframe
...
'''
"""
if model_type == "PyTorch":
(
df_train_and_test,
end_tensor,
forecast_history,
forecast_start_idx,
test_data,
df_predictions,
# df_prediction_samples_std_dev,
) = infer_on_torch_model(model, **inference_params)
# To-do turn this into a general function
g_loss = False
probablistic = True if "probabilistic" in inference_params else False
if isinstance(end_tensor, tuple) and not probablistic:
end_tensor_0 = end_tensor[1]
end_tensor = end_tensor[0]
g_loss = True
if test_data.scale:
print("Un-transforming data")
if probablistic:
print('probabilistic running on infer_on_torch_model')
end_tensor_mean = test_data.inverse_scale(
end_tensor[0].detach().reshape(-1, 1))
end_tensor_list = flatten_list_function(
end_tensor_mean.numpy().tolist())
end_tensor_mean = end_tensor_mean.squeeze(1)
else:
if "n_targets" in model.params:
if model.params["model_name"] == "Informer":
end_tensor = end_tensor[:, :,
0:model.params["n_targets"]]
end_tensor = test_data.inverse_scale(end_tensor.detach())
else:
end_tensor = test_data.inverse_scale(
end_tensor.detach().reshape(-1, 1))
end_tensor_list = flatten_list_function(
end_tensor.numpy().tolist())
end_tensor = end_tensor.squeeze(
1) # Removing extra dim from reshape?
history_length = model.params["dataset_params"]["forecast_history"]
if "n_targets" in model.params:
df_train_and_test.loc[
df_train_and_test.index[history_length:],
"preds"] = end_tensor[:, 0].numpy().tolist()
for i, target in enumerate(target_col):
df_train_and_test["pred_" + target] = 0
df_train_and_test.loc[
df_train_and_test.index[history_length:],
"pred_" + target] = end_tensor[:, i].numpy().tolist()
else:
df_train_and_test.loc[df_train_and_test.index[history_length:],
"preds"] = end_tensor_list
df_train_and_test["pred_" + target_col[0]] = 0
df_train_and_test.loc[df_train_and_test.index[history_length:],
"pred_" +
target_col[0]] = end_tensor_list
print("Current historical dataframe ")
print(df_train_and_test)
for evaluation_metric in model.crit:
idx = 0
for target in target_col:
labels = torch.from_numpy(
df_train_and_test[target][forecast_history:].to_numpy())
evaluation_metric_function = evaluation_metric
if "probabilistic" in inference_params:
s = evaluation_metric_function(
torch.distributions.Normal(end_tensor[0],
end_tensor[1][0]),
labels,
)
elif isinstance(evaluation_metric_function, MASELoss):
s = evaluation_metric_function(
labels, end_tensor,
torch.from_numpy(df_train_and_test[target]
[:forecast_history].to_numpy()))
elif g_loss:
g = GaussianLoss(end_tensor.unsqueeze(1),
end_tensor_0.unsqueeze(1))
s = g(labels.unsqueeze(1))
else:
if "n_targets" in model.params:
s = evaluation_metric_function(
labels,
end_tensor[:, idx],
)
else:
s = evaluation_metric_function(
labels,
end_tensor,
)
idx += 1
eval_log[target + "_" + evaluation_metric.__class__.__name__] = s
# Explain model behaviour using shap
if "probabilistic" in inference_params:
print("Probabilistic explainability currently not supported.")
elif "n_targets" in model.params:
print("Multitask forecasting support coming soon")
elif g_loss:
print("SHAP not yet supported for these models with multiple outputs")
else:
deep_explain_model_summary_plot(model, test_data,
inference_params["datetime_start"])
deep_explain_model_heatmap(model, test_data,
inference_params["datetime_start"])
return eval_log, df_train_and_test, forecast_start_idx, df_predictions
