def make_evaluation_predictions( dataset: Dataset, predictor: Predictor, num_eval_samples: int ) -> Tuple[Iterator[Forecast], Iterator[pd.Series]]: """ Return predictions on the last portion of predict_length time units of the target. Such portion is cut before making predictions, such a function can be used in evaluations where accuracy is evaluated on the last portion of the target. Parameters ---------- dataset Dataset where the evaluation will happen. Only the portion excluding the prediction_length portion is used when making prediction. predictor Model used to draw predictions. num_eval_samples Number of samples to draw on the model when evaluating. Returns ------- """ prediction_length = predictor.prediction_length freq = predictor.freq def add_ts_dataframe(data_iterator: Iterator[DataEntry]) -> DataEntry: for data_entry in data_iterator: data = data_entry.copy() index = pd.date_range( start=data["start"], freq=freq, periods=data["target"].shape[-1], ) data["ts"] = pd.DataFrame( index=index, data=data["target"].transpose() ) yield data def ts_iter(dataset: Dataset) -> pd.DataFrame: for data_entry in add_ts_dataframe(iter(dataset)): yield data_entry["ts"] def truncate_target(data): data = data.copy() target = data["target"] assert ( target.shape[-1] >= prediction_length ) # handles multivariate case (target_dim, history_length) data["target"] = target[..., :-prediction_length] return data # TODO filter out time series with target shorter than prediction length # TODO or fix the evaluator so it supports missing values instead (all # TODO the test set may be gone otherwise with such a filtering) dataset_trunc = TransformedDataset( dataset, transformations=[transform.AdhocTransform(truncate_target)] ) return ( predictor.predict(dataset_trunc, num_eval_samples=num_eval_samples), ts_iter(dataset), )
def make_evaluation_predictions( dataset: Dataset, predictor: Predictor, num_samples: int = 100, ) -> Tuple[Iterator[Forecast], Iterator[pd.Series]]: """ Returns predictions for the trailing prediction_length observations of the given time series, using the given predictor. The predictor will take as input the given time series without the trailing prediction_length observations. Parameters ---------- dataset Dataset where the evaluation will happen. Only the portion excluding the prediction_length portion is used when making prediction. predictor Model used to draw predictions. num_samples Number of samples to draw on the model when evaluating. Only sampling-based models will use this. Returns ------- Tuple[Iterator[Forecast], Iterator[pd.Series]] A pair of iterators, the first one yielding the forecasts, and the second one yielding the corresponding ground truth series. """ prediction_length = predictor.prediction_length freq = predictor.freq lead_time = predictor.lead_time def add_ts_dataframe( data_iterator: Iterator[DataEntry], ) -> Iterator[DataEntry]: for data_entry in data_iterator: data = data_entry.copy() index = pd.date_range( start=data["start"], freq=freq, periods=data["target"].shape[-1], ) data["ts"] = pd.DataFrame(index=index, data=data["target"].transpose()) yield data def ts_iter(dataset: Dataset) -> pd.DataFrame: for data_entry in add_ts_dataframe(iter(dataset)): yield data_entry["ts"] def truncate_target(data): data = data.copy() target = data["target"] assert (target.shape[-1] >= prediction_length ) # handles multivariate case (target_dim, history_length) data["target"] = target[..., :-prediction_length - lead_time] return data # TODO filter out time series with target shorter than prediction length # TODO or fix the evaluator so it supports missing values instead (all # TODO the test set may be gone otherwise with such a filtering) dataset_trunc = TransformedDataset( dataset, transformation=transform.AdhocTransform(truncate_target)) return ( predictor.predict(dataset_trunc, num_samples=num_samples), ts_iter(dataset), )