def InferenceDataLoader(
dataset: Dataset,
*,
transform: Transformation = Identity(),
batch_size: int,
stack_fn: Callable,
):
"""Construct an iterator of batches for inference purposes.
Parameters
----------
dataset
Data to iterate over.
transform
Transformation to be lazily applied as data is being iterated.
The transformation is applied in "inference mode" (``is_train=False``).
batch_size
Number of entries to include in a batch.
stack_fn
Function to use to stack data entries into batches.
This can be used to set a specific array type or computing device
the arrays should end up onto (CPU, GPU).
Returns
-------
Iterable[DataBatch]
An iterable sequence of batches.
"""
transform += Batch(batch_size=batch_size) + AdhocTransform(stack_fn)
return transform.apply(dataset, is_train=False)
def ValidationDataLoader(
dataset: Dataset,
*,
transform: Transformation = Identity(),
batch_size: int,
stack_fn: Callable,
num_prefetch: Optional[int] = None,
num_workers: Optional[int] = None,
decode_fn: Callable = lambda x: x,
):
"""
Construct an iterator of batches for validation purposes.
Parameters
----------
dataset
Data to iterate over.
transform
Transformation to be lazily applied as data is being iterated.
The transformation is applied in "training mode" (``is_train=True``).
batch_size
Number of entries to include in a batch.
stack_fn
Function to use to stack data entries into batches.
This can be used to set a specific array type or computing device
the arrays should end up onto (CPU, GPU).
num_workers
Number of worker processes to use. Default: None.
num_prefetch
Sets the length of the queue of batches being produced by worker
processes. (Only meaningful when ``num_workers is not None``).
decode_fn
A function called on each batch after it's been taken out of the queue.
(Only meaningful when ``num_workers is not None``).
Returns
-------
Iterable[DataBatch]
An iterable sequence of batches.
"""
transform += Batch(batch_size=batch_size) + AdhocTransform(stack_fn)
transformed_dataset = transform.apply(dataset, is_train=True)
if num_workers is None:
return transformed_dataset
return MultiProcessLoader(
transformed_dataset,
decode_fn=decode_fn,
num_workers=num_workers,
max_queue_size=num_prefetch,
)
def TrainDataLoader(
dataset: Dataset,
*,
transform: Transformation = Identity(),
batch_size: int,
stack_fn: Callable,
num_batches_per_epoch: Optional[int] = None,
num_prefetch: Optional[int] = None,
num_workers: Optional[int] = None,
shuffle_buffer_length: Optional[int] = None,
decode_fn: Callable = lambda x: x,
):
"""Construct an iterator of batches for training purposes.
This function wraps around ``DataLoader`` to offer training-specific
behaviour and options, as follows:
1. The provided dataset is iterated cyclically, so that one can go over
it multiple times in a single epoch. 2. A transformation must be
provided, that is lazily applied as the dataset is being iterated;
this is useful e.g. to slice random instances of fixed length out of
each time series in the dataset. 3. The resulting batches can be
iterated in a pseudo-shuffled order.
The returned object is a stateful iterator, whose length is either
``num_batches_per_epoch`` (if not ``None``) or infinite (otherwise).
Parameters
----------
dataset
Data to iterate over.
transform
Transformation to be lazily applied as data is being iterated.
The transformation is applied in "training mode" (``is_train=True``).
batch_size
Number of entries to include in a batch.
stack_fn
Function to use to stack data entries into batches.
This can be used to set a specific array type or computing device
the arrays should end up onto (CPU, GPU).
num_batches_per_epoch
Length of the iterator. If ``None``, then the iterator is endless.
num_workers
Number of worker processes to use. Default: None.
num_prefetch
Sets the length of the queue of batches being produced by worker
processes. (Only meaningful when ``num_workers is not None``).
shuffle_buffer_length
Size of the buffer used for shuffling. Default: None, in which case no
shuffling occurs.
decode_fn
A function called on each batch after it's been taken out of the queue.
(Only meaningful when ``num_workers is not None``).
Returns
-------
Iterator[DataBatch]
An iterator of batches.
"""
dataset = Cyclic(dataset)
if shuffle_buffer_length:
dataset = PseudoShuffled(dataset, shuffle_buffer_length)
transform += Batch(batch_size=batch_size) + AdhocTransform(stack_fn)
transformed_dataset = transform.apply(dataset, is_train=True)
if num_workers is not None:
loader = MultiProcessLoader(
transformed_dataset,
decode_fn=decode_fn,
num_workers=num_workers,
max_queue_size=num_prefetch,
)
batches = iter(loader)
else:
batches = iter(transformed_dataset)
if num_batches_per_epoch is None:
return batches
else:
return IterableSlice(batches, num_batches_per_epoch)
def make_evaluation_predictions(
dataset: Dataset,
predictor: Predictor,
num_samples: int = 100,
) -> Tuple[Iterator[Forecast], Iterator[pd.Series]]:
"""
Returns predictions for the trailing prediction_length observations of the
given time series, using the given predictor.
The predictor will take as input the given time series without the trailing
prediction_length observations.
Parameters
----------
dataset
Dataset where the evaluation will happen. Only the portion excluding
the prediction_length portion is used when making prediction.
predictor
Model used to draw predictions.
num_samples
Number of samples to draw on the model when evaluating. Only
sampling-based models will use this.
Returns
-------
Tuple[Iterator[Forecast], Iterator[pd.Series]]
A pair of iterators, the first one yielding the forecasts, and the
second one yielding the corresponding ground truth series.
"""
prediction_length = predictor.prediction_length
lead_time = predictor.lead_time
def add_ts_dataframe(
data_iterator: Iterator[DataEntry], ) -> Iterator[DataEntry]:
for data_entry in data_iterator:
data = data_entry.copy()
index = pd.period_range(
start=data[FieldName.START],
periods=data[FieldName.TARGET].shape[-1],
)
data["ts"] = pd.DataFrame(index=index,
data=data[FieldName.TARGET].transpose())
yield data
def ts_iter(dataset: Dataset) -> pd.DataFrame:
for data_entry in add_ts_dataframe(iter(dataset)):
yield data_entry["ts"]
def truncate_target(data):
data = data.copy()
target = data[FieldName.TARGET]
assert (target.shape[-1] >= prediction_length
) # handles multivariate case (target_dim, history_length)
data[FieldName.TARGET] = target[..., :-prediction_length - lead_time]
return data
# TODO filter out time series with target shorter than prediction length
# TODO or fix the evaluator so it supports missing values instead (all
# TODO the test set may be gone otherwise with such a filtering)
dataset_trunc = AdhocTransform(truncate_target).apply(dataset)
return (
predictor.predict(dataset_trunc, num_samples=num_samples),
ts_iter(dataset),
)