def check_dataloader_output(dataset: TimeSeriesDataSet,
out: Dict[str, torch.Tensor]):
x, y = out
assert isinstance(y,
tuple), "y output should be tuple of wegith and target"
# check for nans and finite
for k, v in x.items():
for vi in to_list(v):
assert torch.isfinite(vi).all(), f"Values for {k} should be finite"
assert not torch.isnan(
vi).any(), f"Values for {k} should not be nan"
# check weight
assert y[1] is None or isinstance(
y[1], torch.Tensor), "weights should be none or tensor"
if isinstance(y[1], torch.Tensor):
assert torch.isfinite(y[1]).all(), "Values for weight should be finite"
assert not torch.isnan(
y[1]).any(), "Values for weight should not be nan"
# check target
for targeti in to_list(y[0]):
assert torch.isfinite(
targeti).all(), "Values for target should be finite"
assert not torch.isnan(
targeti).any(), "Values for target should not be nan"
# check shape
assert x["encoder_cont"].size(2) == len(dataset.reals)
assert x["encoder_cat"].size(2) == len(dataset.flat_categoricals)
def plot_prediction(
self,
x: Dict[str, torch.Tensor],
out: Dict[str, torch.Tensor],
idx: int,
plot_attention: bool = True,
add_loss_to_title: bool = False,
show_future_observed: bool = True,
ax=None,
**kwargs,
) -> plt.Figure:
"""
Plot actuals vs prediction and attention
Args:
x (Dict[str, torch.Tensor]): network input
out (Dict[str, torch.Tensor]): network output
idx (int): sample index
plot_attention: if to plot attention on secondary axis
add_loss_to_title: if to add loss to title. Default to False.
show_future_observed: if to show actuals for future. Defaults to True.
ax: matplotlib axes to plot on
Returns:
plt.Figure: matplotlib figure
"""
# plot prediction as normal
fig = super().plot_prediction(
x,
out,
idx=idx,
add_loss_to_title=add_loss_to_title,
show_future_observed=show_future_observed,
ax=ax,
**kwargs,
)
# add attention on secondary axis
if plot_attention:
interpretation = self.interpret_output(
out.iget(slice(idx, idx + 1)))
for f in to_list(fig):
ax = f.axes[0]
ax2 = ax.twinx()
ax2.set_ylabel("Attention")
encoder_length = x["encoder_lengths"][0]
ax2.plot(
torch.arange(-encoder_length, 0),
interpretation["attention"][
0, -encoder_length:].detach().cpu(),
alpha=0.2,
color="k",
)
f.tight_layout()
return fig
def __init__(
self,
input_size: int,
output_size: int,
hidden_size: int,
n_hidden_layers: int,
target_sizes: Union[int, List[int]] = [],
**kwargs,
):
# saves arguments in signature to `.hparams` attribute, mandatory call - do not skip this
self.save_hyperparameters()
# pass additional arguments to BaseModel.__init__, mandatory call - do not skip this
super().__init__(**kwargs)
self.network = FullyConnectedModule(
input_size=self.hparams.input_size *
len(to_list(self.hparams.target_sizes)),
output_size=self.hparams.output_size *
sum(to_list(self.hparams.target_sizes)),
hidden_size=self.hparams.hidden_size,
n_hidden_layers=self.hparams.n_hidden_layers,
)
def from_dataset(cls, dataset: TimeSeriesDataSet, **kwargs):
# By default only handle targets of size one here, categorical targets would be of larger size
new_kwargs = {
"target_sizes": [1] * len(to_list(dataset.target)),
"output_size": dataset.max_prediction_length,
"input_size": dataset.max_encoder_length,
}
new_kwargs.update(
kwargs
) # use to pass real hyperparameters and override defaults set by dataset
# example for dataset validation
assert dataset.max_prediction_length == dataset.min_prediction_length, "Decoder only supports a fixed length"
assert dataset.min_encoder_length == dataset.max_encoder_length, "Encoder only supports a fixed length"
assert (len(dataset.time_varying_known_categoricals) == 0
and len(dataset.time_varying_known_reals) == 0
and len(dataset.time_varying_unknown_categoricals) == 0
and len(dataset.static_categoricals) == 0
and len(dataset.static_reals) == 0
and len(dataset.time_varying_unknown_reals) == len(
dataset.target_names
) # Expect as as many unknown reals as targets
), "Only covariate should be in 'time_varying_unknown_reals'"
return super().from_dataset(dataset, **new_kwargs)
def __init__(
self,
cell_type: str = "LSTM",
hidden_size: int = 10,
rnn_layers: int = 2,
dropout: float = 0.1,
static_categoricals: List[str] = [],
static_reals: List[str] = [],
time_varying_categoricals_encoder: List[str] = [],
time_varying_categoricals_decoder: List[str] = [],
categorical_groups: Dict[str, List[str]] = {},
time_varying_reals_encoder: List[str] = [],
time_varying_reals_decoder: List[str] = [],
embedding_sizes: Dict[str, Tuple[int, int]] = {},
embedding_paddings: List[str] = [],
embedding_labels: Dict[str, np.ndarray] = {},
x_reals: List[str] = [],
x_categoricals: List[str] = [],
output_size: Union[int, List[int]] = 1,
target: Union[str, List[str]] = None,
target_lags: Dict[str, List[int]] = {},
loss: MultiHorizonMetric = None,
logging_metrics: nn.ModuleList = None,
**kwargs,
):
"""
Recurrent Network.
Simple LSTM or GRU layer followed by output layer
Args:
cell_type (str, optional): Recurrent cell type ["LSTM", "GRU"]. Defaults to "LSTM".
hidden_size (int, optional): hidden recurrent size - the most important hyperparameter along with
``rnn_layers``. Defaults to 10.
rnn_layers (int, optional): Number of RNN layers - important hyperparameter. Defaults to 2.
dropout (float, optional): Dropout in RNN layers. Defaults to 0.1.
static_categoricals: integer of positions of static categorical variables
static_reals: integer of positions of static continuous variables
time_varying_categoricals_encoder: integer of positions of categorical variables for encoder
time_varying_categoricals_decoder: integer of positions of categorical variables for decoder
time_varying_reals_encoder: integer of positions of continuous variables for encoder
time_varying_reals_decoder: integer of positions of continuous variables for decoder
categorical_groups: dictionary where values
are list of categorical variables that are forming together a new categorical
variable which is the key in the dictionary
x_reals: order of continuous variables in tensor passed to forward function
x_categoricals: order of categorical variables in tensor passed to forward function
embedding_sizes: dictionary mapping (string) indices to tuple of number of categorical classes and
embedding size
embedding_paddings: list of indices for embeddings which transform the zero's embedding to a zero vector
embedding_labels: dictionary mapping (string) indices to list of categorical labels
output_size (Union[int, List[int]], optional): number of outputs (e.g. number of quantiles for
QuantileLoss and one target or list of output sizes).
target (str, optional): Target variable or list of target variables. Defaults to None.
target_lags (Dict[str, Dict[str, int]]): dictionary of target names mapped to list of time steps by
which the variable should be lagged.
Lags can be useful to indicate seasonality to the models. If you know the seasonalit(ies) of your data,
add at least the target variables with the corresponding lags to improve performance.
Defaults to no lags, i.e. an empty dictionary.
loss (MultiHorizonMetric, optional): loss: loss function taking prediction and targets.
logging_metrics (nn.ModuleList, optional): Metrics to log during training.
Defaults to nn.ModuleList([SMAPE(), MAE(), RMSE(), MAPE(), MASE()]).
"""
if loss is None:
loss = MAE()
if logging_metrics is None:
logging_metrics = nn.ModuleList(
[SMAPE(), MAE(), RMSE(),
MAPE(), MASE()])
self.save_hyperparameters()
# store loss function separately as it is a module
super().__init__(loss=loss, logging_metrics=logging_metrics, **kwargs)
self.embeddings = MultiEmbedding(
embedding_sizes=embedding_sizes,
embedding_paddings=embedding_paddings,
categorical_groups=categorical_groups,
x_categoricals=x_categoricals,
)
lagged_target_names = [
l for lags in target_lags.values() for l in lags
]
assert set(self.encoder_variables) - set(
to_list(target)
) - set(lagged_target_names) == set(self.decoder_variables) - set(
lagged_target_names
), "Encoder and decoder variables have to be the same apart from target variable"
for targeti in to_list(target):
assert (
targeti in time_varying_reals_encoder
), f"target {targeti} has to be real" # todo: remove this restriction
assert (
isinstance(target, str) and isinstance(loss, MultiHorizonMetric)
) or (
isinstance(target, (list, tuple)) and isinstance(loss, MultiLoss)
and len(loss) == len(target)
), "number of targets should be equivalent to number of loss metrics"
rnn_class = get_rnn(cell_type)
cont_size = len(self.reals)
cat_size = sum(
[size[1] for size in self.hparams.embedding_sizes.values()])
input_size = cont_size + cat_size
self.rnn = rnn_class(
input_size=input_size,
hidden_size=self.hparams.hidden_size,
num_layers=self.hparams.rnn_layers,
dropout=self.hparams.dropout if self.hparams.rnn_layers > 1 else 0,
batch_first=True,
)
# add linear layers for argument projects
if isinstance(target, str): # single target
self.output_projector = nn.Linear(self.hparams.hidden_size,
self.hparams.output_size)
assert not isinstance(
self.loss, QuantileLoss
), "QuantileLoss does not work with recurrent network"
else: # multi target
self.output_projector = nn.ModuleList([
nn.Linear(self.hparams.hidden_size, size)
for size in self.hparams.output_size
])
for l in self.loss:
assert not isinstance(
l, QuantileLoss
), "QuantileLoss does not work with recurrent network"
def __init__(
self,
cell_type: str = "LSTM",
hidden_size: int = 10,
rnn_layers: int = 2,
dropout: float = 0.1,
static_categoricals: List[str] = [],
static_reals: List[str] = [],
time_varying_categoricals_encoder: List[str] = [],
time_varying_categoricals_decoder: List[str] = [],
categorical_groups: Dict[str, List[str]] = {},
time_varying_reals_encoder: List[str] = [],
time_varying_reals_decoder: List[str] = [],
embedding_sizes: Dict[str, Tuple[int, int]] = {},
embedding_paddings: List[str] = [],
embedding_labels: Dict[str, np.ndarray] = {},
x_reals: List[str] = [],
x_categoricals: List[str] = [],
n_validation_samples: int = None,
n_plotting_samples: int = None,
target: Union[str, List[str]] = None,
loss: MultiHorizonMetric = None,
logging_metrics: nn.ModuleList = None,
**kwargs,
):
"""
Args:
cell_type (str, optional): Recurrent cell type ["LSTM", "GRU"]. Defaults to "LSTM".
hidden_size (int, optional): hidden recurrent size - the most important hyperparameter along with
``rnn_layers``. Defaults to 10.
rnn_layers (int, optional): Number of RNN layers - important hyperparameter. Defaults to 2.
dropout (float, optional): Dropout in RNN layers. Defaults to 0.1.
static_categoricals: integer of positions of static categorical variables
static_reals: integer of positions of static continuous variables
time_varying_categoricals_encoder: integer of positions of categorical variables for encoder
time_varying_categoricals_decoder: integer of positions of categorical variables for decoder
time_varying_reals_encoder: integer of positions of continuous variables for encoder
time_varying_reals_decoder: integer of positions of continuous variables for decoder
categorical_groups: dictionary where values
are list of categorical variables that are forming together a new categorical
variable which is the key in the dictionary
x_reals: order of continuous variables in tensor passed to forward function
x_categoricals: order of categorical variables in tensor passed to forward function
embedding_sizes: dictionary mapping (string) indices to tuple of number of categorical classes and
embedding size
embedding_paddings: list of indices for embeddings which transform the zero's embedding to a zero vector
embedding_labels: dictionary mapping (string) indices to list of categorical labels
n_validation_samples (int, optional): Number of samples to use for calculating validation metrics.
Defaults to None, i.e. no sampling at validation stage and using "mean" of distribution for logging
metrics calculation.
n_plotting_samples (int, optional): Number of samples to generate for plotting predictions
during training. Defaults to ``n_validation_samples`` if not None or 100 otherwise.
target (str, optional): Target variable or list of target variables. Defaults to None.
loss (DistributionLoss, optional): Distribution loss function. Keep in mind that each distribution
loss function might have specific requirements for target normalization.
Defaults to :py:class:`~pytorch_forecasting.metrics.NormalDistributionLoss`.
logging_metrics (nn.ModuleList, optional): Metrics to log during training.
Defaults to nn.ModuleList([SMAPE(), MAE(), RMSE(), MAPE(), MASE()]).
"""
# saves arguments in signature to `.hparams` attribute, mandatory call - do not skip this
self.save_hyperparameters()
# pass additional arguments to BaseModel.__init__, mandatory call - do not skip this
super().__init__(loss=loss, logging_metrics=logging_metrics, **kwargs)
assert set(self.encoder_variables) - set(to_list(target)) == set(
self.decoder_variables
), "Encoder and decoder variables have to be the same apart from target variable"
for targeti in to_list(target):
assert (
targeti in time_varying_reals_encoder
), f"target {targeti} has to be real" # todo: remove this restriction
self.embeddings = MultiEmbedding(
embedding_sizes=embedding_sizes,
embedding_paddings=embedding_paddings,
categorical_groups=categorical_groups,
x_categoricals=x_categoricals,
)
time_series_rnn = get_rnn(cell_type)
cont_size = len(self.reals)
cat_size = sum([size[1] for size in self.hparams.embedding_sizes.values()])
input_size = cont_size + cat_size
self.rnn = time_series_rnn(
input_size=input_size,
hidden_size=self.hparams.hidden_size,
num_layers=self.hparams.rnn_layers,
dropout=self.hparams.dropout if self.hparams.rnn_layers > 1 else 0,
batch_first=True,
)
# add linear layers for argument projects
if isinstance(target, str): # single target
self.output_projector = nn.Linear(self.hparams.hidden_size, 1)
else: # multi target
self.output_projector = nn.ModuleList([nn.Linear(self.hparams.hidden_size, 1) for _ in target])
def __init__(
self,
cell_type: str = "LSTM",
hidden_size: int = 10,
rnn_layers: int = 2,
dropout: float = 0.1,
static_categoricals: List[str] = [],
static_reals: List[str] = [],
time_varying_categoricals_encoder: List[str] = [],
time_varying_categoricals_decoder: List[str] = [],
categorical_groups: Dict[str, List[str]] = {},
time_varying_reals_encoder: List[str] = [],
time_varying_reals_decoder: List[str] = [],
embedding_sizes: Dict[str, Tuple[int, int]] = {},
embedding_paddings: List[str] = [],
embedding_labels: Dict[str, np.ndarray] = {},
x_reals: List[str] = [],
x_categoricals: List[str] = [],
n_validation_samples: int = None,
n_plotting_samples: int = None,
target: Union[str, List[str]] = None,
loss: DistributionLoss = None,
logging_metrics: nn.ModuleList = None,
**kwargs,
):
"""
DeepAR Network.
The code is based on the article `DeepAR: Probabilistic forecasting with autoregressive recurrent networks
<https://www.sciencedirect.com/science/article/pii/S0169207019301888>`_.
Args:
cell_type (str, optional): Recurrent cell type ["LSTM", "GRU"]. Defaults to "LSTM".
hidden_size (int, optional): hidden recurrent size - the most important hyperparameter along with
``rnn_layers``. Defaults to 10.
rnn_layers (int, optional): Number of RNN layers - important hyperparameter. Defaults to 2.
dropout (float, optional): Dropout in RNN layers. Defaults to 0.1.
static_categoricals: integer of positions of static categorical variables
static_reals: integer of positions of static continuous variables
time_varying_categoricals_encoder: integer of positions of categorical variables for encoder
time_varying_categoricals_decoder: integer of positions of categorical variables for decoder
time_varying_reals_encoder: integer of positions of continuous variables for encoder
time_varying_reals_decoder: integer of positions of continuous variables for decoder
categorical_groups: dictionary where values
are list of categorical variables that are forming together a new categorical
variable which is the key in the dictionary
x_reals: order of continuous variables in tensor passed to forward function
x_categoricals: order of categorical variables in tensor passed to forward function
embedding_sizes: dictionary mapping (string) indices to tuple of number of categorical classes and
embedding size
embedding_paddings: list of indices for embeddings which transform the zero's embedding to a zero vector
embedding_labels: dictionary mapping (string) indices to list of categorical labels
n_validation_samples (int, optional): Number of samples to use for calculating validation metrics.
Defaults to None, i.e. no sampling at validation stage and using "mean" of distribution for logging
metrics calculation.
n_plotting_samples (int, optional): Number of samples to generate for plotting predictions
during training. Defaults to ``n_validation_samples`` if not None or 100 otherwise.
target (str, optional): Target variable or list of target variables. Defaults to None.
loss (DistributionLoss, optional): Distribution loss function. Keep in mind that each distribution
loss function might have specific requirements for target normalization.
Defaults to :py:class:`~pytorch_forecasting.metrics.NormalDistributionLoss`.
logging_metrics (nn.ModuleList, optional): Metrics to log during training.
Defaults to nn.ModuleList([SMAPE(), MAE(), RMSE(), MAPE(), MASE()]).
"""
if loss is None:
loss = NormalDistributionLoss()
if logging_metrics is None:
logging_metrics = nn.ModuleList(
[SMAPE(), MAE(), RMSE(),
MAPE(), MASE()])
if n_plotting_samples is None:
if n_validation_samples is None:
n_plotting_samples = n_validation_samples
else:
n_plotting_samples = 100
self.save_hyperparameters()
# store loss function separately as it is a module
super().__init__(loss=loss, logging_metrics=logging_metrics, **kwargs)
self.embeddings = MultiEmbedding(
embedding_sizes=embedding_sizes,
embedding_paddings=embedding_paddings,
categorical_groups=categorical_groups,
x_categoricals=x_categoricals,
)
assert set(self.encoder_variables) - set(to_list(target)) == set(
self.decoder_variables
), "Encoder and decoder variables have to be the same apart from target variable"
for targeti in to_list(target):
assert (
targeti in time_varying_reals_encoder
), f"target {targeti} has to be real" # todo: remove this restriction
assert (
isinstance(target, str) and isinstance(loss, DistributionLoss)
) or (
isinstance(target,
(list, tuple)) and isinstance(loss, MultiLoss) and
len(loss)
== len(target)
), "number of targets should be equivalent to number of loss metrics"
time_series_rnn = get_cell(cell_type)
self.rnn = time_series_rnn(
input_size=self.input_size,
hidden_size=self.hparams.hidden_size,
num_layers=self.hparams.rnn_layers,
dropout=self.hparams.dropout if self.hparams.rnn_layers > 1 else 0,
batch_first=True,
)
# add linear layers for argument projects
if isinstance(loss, MultiLoss): # multi target
self.distribution_projector = nn.ModuleList([
nn.Linear(self.hparams.hidden_size, len(args))
for args in self.loss.distribution_arguments
])
else:
self.distribution_projector = nn.Linear(
self.hparams.hidden_size,
len(self.loss.distribution_arguments))
def target_positions(self):
return torch.tensor([
self.hparams.x_reals.index(name)
for name in to_list(self.hparams.target)
],
device=self.device)
