def make_evaluation_predictions(
dataset: Dataset, predictor: Predictor, num_eval_samples: int
) -> Tuple[Iterator[Forecast], Iterator[pd.Series]]:
"""
Return predictions on the last portion of predict_length time units of the
target. Such portion is cut before making predictions, such a function can
be used in evaluations where accuracy is evaluated on the last portion of
the target.
Parameters
----------
dataset
Dataset where the evaluation will happen. Only the portion excluding
the prediction_length portion is used when making prediction.
predictor
Model used to draw predictions.
num_eval_samples
Number of samples to draw on the model when evaluating.
Returns
-------
"""
prediction_length = predictor.prediction_length
freq = predictor.freq
def add_ts_dataframe(data_iterator: Iterator[DataEntry]) -> DataEntry:
for data_entry in data_iterator:
data = data_entry.copy()
index = pd.date_range(
start=data["start"],
freq=freq,
periods=data["target"].shape[-1],
)
data["ts"] = pd.DataFrame(
index=index, data=data["target"].transpose()
)
yield data
def ts_iter(dataset: Dataset) -> pd.DataFrame:
for data_entry in add_ts_dataframe(iter(dataset)):
yield data_entry["ts"]
def truncate_target(data):
data = data.copy()
target = data["target"]
assert (
target.shape[-1] >= prediction_length
) # handles multivariate case (target_dim, history_length)
data["target"] = target[..., :-prediction_length]
return data
# TODO filter out time series with target shorter than prediction length
# TODO or fix the evaluator so it supports missing values instead (all
# TODO the test set may be gone otherwise with such a filtering)
dataset_trunc = TransformedDataset(
dataset, transformations=[transform.AdhocTransform(truncate_target)]
)
return (
predictor.predict(dataset_trunc, num_eval_samples=num_eval_samples),
ts_iter(dataset),
)
def make_evaluation_predictions(
dataset: Dataset,
predictor: Predictor,
num_samples: int = 100,
) -> Tuple[Iterator[Forecast], Iterator[pd.Series]]:
"""
Returns predictions for the trailing prediction_length observations of the given
time series, using the given predictor.
The predictor will take as input the given time series without the trailing
prediction_length observations.
Parameters
----------
dataset
Dataset where the evaluation will happen. Only the portion excluding
the prediction_length portion is used when making prediction.
predictor
Model used to draw predictions.
num_samples
Number of samples to draw on the model when evaluating. Only sampling-based
models will use this.
Returns
-------
Tuple[Iterator[Forecast], Iterator[pd.Series]]
A pair of iterators, the first one yielding the forecasts, and the second
one yielding the corresponding ground truth series.
"""
prediction_length = predictor.prediction_length
freq = predictor.freq
lead_time = predictor.lead_time
def add_ts_dataframe(
data_iterator: Iterator[DataEntry], ) -> Iterator[DataEntry]:
for data_entry in data_iterator:
data = data_entry.copy()
index = pd.date_range(
start=data["start"],
freq=freq,
periods=data["target"].shape[-1],
)
data["ts"] = pd.DataFrame(index=index,
data=data["target"].transpose())
yield data
def ts_iter(dataset: Dataset) -> pd.DataFrame:
for data_entry in add_ts_dataframe(iter(dataset)):
yield data_entry["ts"]
def truncate_target(data):
data = data.copy()
target = data["target"]
assert (target.shape[-1] >= prediction_length
) # handles multivariate case (target_dim, history_length)
data["target"] = target[..., :-prediction_length - lead_time]
return data
# TODO filter out time series with target shorter than prediction length
# TODO or fix the evaluator so it supports missing values instead (all
# TODO the test set may be gone otherwise with such a filtering)
dataset_trunc = TransformedDataset(
dataset, transformation=transform.AdhocTransform(truncate_target))
return (
predictor.predict(dataset_trunc, num_samples=num_samples),
ts_iter(dataset),
)