def train_model(
self,
training_data: Optional[Dataset] = None,
validation_data: Optional[Dataset] = None,
num_workers: Optional[int] = None,
num_prefetch: Optional[int] = None,
shuffle_buffer_length: Optional[int] = None,
cache_data: bool = False,
) -> TrainOutput:
transformation = self.create_transformation()
transformed_training_data = TransformedDataset(
training_data, transformation
)
training_data_loader = self.create_training_data_loader(
transformed_training_data
if not cache_data
else Cached(transformed_training_data),
num_workers=num_workers,
num_prefetch=num_prefetch,
shuffle_buffer_length=shuffle_buffer_length,
)
validation_data_loader = None
if validation_data is not None:
transformed_validation_data = TransformedDataset(
validation_data, transformation
)
validation_data_loader = self.create_validation_data_loader(
transformed_validation_data
if not cache_data
else Cached(transformed_validation_data),
num_workers=num_workers,
)
training_network = self.create_training_network()
self.trainer(
net=training_network,
train_iter=training_data_loader,
validation_iter=validation_data_loader,
)
with self.trainer.ctx:
predictor = self.create_predictor(transformation, training_network)
return TrainOutput(
transformation=transformation,
trained_net=training_network,
predictor=predictor,
)
def run_test(self, dataset, estimator, predictor):
test_dataset = TransformedDataset(
dataset,
transformations=[
FilterTransformation(lambda el: el['target'].shape[-1] >
predictor.prediction_length)
],
)
len_orig = len(dataset)
len_filtered = len(test_dataset)
if len_orig > len_filtered:
logging.warning(
'Not all time-series in the test-channel have '
'enough data to be used for evaluation. Proceeding with '
f'{len_filtered}/{len_orig} '
f'(~{int(len_filtered/len_orig*100)}%) items.')
try:
log.metric('test_dataset_stats', test_dataset.calc_stats())
except GluonTSDataError as error:
logging.error(
f"Failure whilst calculating stats for test dataset: {error}")
return
if isinstance(estimator, GluonEstimator) and isinstance(
predictor, GluonPredictor):
inference_data_loader = InferenceDataLoader(
dataset=test_dataset,
transform=predictor.input_transform,
batch_size=estimator.trainer.batch_size,
ctx=estimator.trainer.ctx,
float_type=estimator.float_type,
)
if estimator.trainer.hybridize:
predictor.hybridize(batch=next(iter(inference_data_loader)))
if self.hyperparameters.get('use_symbol_block_predictor'):
predictor = predictor.as_symbol_block_predictor(
batch=next(iter(inference_data_loader)))
num_eval_samples = self.hyperparameters.get('num_eval_samples', 100)
quantiles = self.hyperparameters.get(
'quantiles', (0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9))
# we only log aggregate metrics for now as item metrics may be
# very large
predictions, input_timeseries = backtest.make_evaluation_predictions(
test_dataset, predictor, num_eval_samples)
agg_metrics, _item_metrics = Evaluator(quantiles=quantiles)(
input_timeseries, predictions, num_series=len_filtered)
log.metric("agg_metrics", agg_metrics)
def run_test(env: TrainEnv, predictor: Predictor,
test_dataset: Dataset) -> None:
len_original = len(test_dataset)
test_dataset = TransformedDataset(
base_dataset=test_dataset,
transformations=[
FilterTransformation(
lambda x: x["target"].shape[-1] > predictor.prediction_length)
],
)
len_filtered = len(test_dataset)
if len_original > len_filtered:
logger.warning(
f"Not all time-series in the test-channel have "
f"enough data to be used for evaluation. Proceeding with "
f"{len_filtered}/{len_original} "
f"(~{int(len_filtered / len_original * 100)}%) items.")
forecast_it, ts_it = backtest.make_evaluation_predictions(
dataset=test_dataset, predictor=predictor, num_samples=100)
agg_metrics, _item_metrics = Evaluator()(
ts_iterator=ts_it,
fcst_iterator=forecast_it,
num_series=len(test_dataset),
)
# we only log aggregate metrics for now as item metrics may be very large
for name, score in agg_metrics.items():
logger.info(f"#test_score ({env.current_host}, {name}): {score}")
def TrainDataLoader(
dataset: Dataset,
*,
transform: Transformation,
batch_size: int,
stack_fn: Callable,
num_batches_per_epoch: Optional[int] = None,
num_workers: Optional[int] = None,
num_prefetch: Optional[int] = None,
shuffle_buffer_length: Optional[int] = None,
decode_fn: Callable = lambda x: x,
):
transformed_dataset = TransformedDataset(Cyclic(dataset),
transform,
is_train=True)
data_iterable = (PseudoShuffled(
transformed_dataset, shuffle_buffer_length=shuffle_buffer_length)
if shuffle_buffer_length is not None else
transformed_dataset)
data_loader = DataLoader(
data_iterable=data_iterable,
batch_size=batch_size,
stack_fn=stack_fn,
num_workers=num_workers,
num_prefetch=num_prefetch,
decode_fn=decode_fn,
)
return (iter(data_loader) if num_batches_per_epoch is None else
IterableSlice(iter(data_loader), num_batches_per_epoch))
def InferenceDataLoader(
dataset: Dataset,
*,
transform: Transformation,
batch_size: int,
stack_fn: Callable,
):
"""Construct an iterator of batches for inference purposes.
Parameters
----------
dataset
Data to iterate over.
transform
Transformation to be lazily applied as data is being iterated.
The transformation is applied in "inference mode" (``is_train=False``).
batch_size
Number of entries to include in a batch.
stack_fn
Function to use to stack data entries into batches.
This can be used to set a specific array type or computing device
the arrays should end up onto (CPU, GPU).
Returns
-------
Iterable[DataBatch]
An iterable sequence of batches.
"""
return DataLoader(
data_iterable=TransformedDataset(dataset, transform, is_train=False),
batch_size=batch_size,
stack_fn=stack_fn,
)
def run_test(
env: TrainEnv, predictor: Predictor, test_dataset: Dataset
) -> None:
len_original = maybe_len(test_dataset)
test_dataset = TransformedDataset(
test_dataset,
FilterTransformation(
lambda x: x["target"].shape[-1] > predictor.prediction_length
),
)
len_filtered = len(test_dataset)
if len_original is not None and len_original > len_filtered:
logger.warning(
f"Not all time-series in the test-channel have "
f"enough data to be used for evaluation. Proceeding with "
f"{len_filtered}/{len_original} "
f"(~{int(len_filtered / len_original * 100)}%) items."
)
forecast_it, ts_it = backtest.make_evaluation_predictions(
dataset=test_dataset, predictor=predictor, num_samples=100
)
if isinstance(predictor, RepresentableBlockPredictor) and isinstance(
predictor.forecast_generator, QuantileForecastGenerator
):
quantiles = predictor.forecast_generator.quantiles
logger.info(f"Using quantiles `{quantiles}` for evaluation.")
evaluator = Evaluator(quantiles=quantiles)
else:
evaluator = Evaluator()
agg_metrics, item_metrics = evaluator(
ts_iterator=ts_it,
fcst_iterator=forecast_it,
num_series=len(test_dataset),
)
# we only log aggregate metrics for now as item metrics may be very large
for name, score in agg_metrics.items():
logger.info(f"#test_score ({env.current_host}, {name}): {score}")
# store metrics
with open(env.path.model / "agg_metrics.json", "w") as agg_metric_file:
json.dump(agg_metrics, agg_metric_file)
with open(env.path.model / "item_metrics.csv", "w") as item_metrics_file:
item_metrics.to_csv(item_metrics_file, index=False)
def InferenceDataLoader(
dataset: Dataset,
*,
transform: Transformation,
batch_size: int,
stack_fn: Callable,
num_workers: Optional[int] = None,
num_prefetch: Optional[int] = None,
shuffle_buffer_length: Optional[int] = None,
):
return DataLoader(
data_iterable=TransformedDataset(dataset, transform, is_train=False),
batch_size=batch_size,
stack_fn=stack_fn,
)
def __init__(
self,
dataset: Dataset,
transform: Transformation,
is_train: bool = True,
shuffle_buffer_length: Optional[int] = None,
cache_data: bool = False,
):
super().__init__()
self.shuffle_buffer_length = shuffle_buffer_length
self.transformed_dataset = TransformedDataset(
Cyclic(dataset) if not cache_data else Cached(Cyclic(dataset)),
transform,
is_train=is_train,
)
def create_validation_data_loader(
self,
data: Dataset,
**kwargs,
) -> DataLoader:
validation_transform = (self._create_instance_splitter("validation") +
self._create_post_split_transform() +
SelectFields(["past_target", "valid_length"]))
return DataLoader(
data_iterable=TransformedDataset(data,
validation_transform,
is_train=True),
batch_size=self.batch_size,
stack_fn=self._stack_fn(),
decode_fn=partial(as_in_context, ctx=self.trainer.ctx),
)
def ValidationDataLoader(
dataset: Dataset,
*,
transform: Transformation,
batch_size: int,
stack_fn: Callable,
num_workers: Optional[int] = None,
num_prefetch: Optional[int] = None,
shuffle_buffer_length: Optional[int] = None,
decode_fn: Callable = lambda x: x,
):
return DataLoader(
data_iterable=TransformedDataset(dataset, transform, is_train=True),
batch_size=batch_size,
stack_fn=stack_fn,
decode_fn=decode_fn,
)
def prepare_test_dataset(dataset: Dataset, prediction_length: int) -> Dataset:
test_dataset = TransformedDataset(
dataset,
transformations=[
FilterTransformation(
lambda el: el['target'].shape[-1] > prediction_length)
],
)
len_orig = len(dataset)
len_filtered = len(test_dataset)
if len_orig > len_filtered:
log.logger.warning(
'Not all time-series in the test-channel have '
'enough data to be used for evaluation. Proceeding with '
f'{len_filtered}/{len_orig} '
f'(~{int(len_filtered / len_orig * 100)}%) items.')
return test_dataset
def construct_training_iterator(
dataset: Dataset,
*,
transform: Transformation,
shuffle_buffer_length: Optional[int] = None,
) -> Iterator[DataEntry]:
transformed_dataset = TransformedDataset(
cyclic(dataset),
transform,
is_train=True,
)
if shuffle_buffer_length is None:
return iter(transformed_dataset)
else:
return pseudo_shuffled(
iter(transformed_dataset),
shuffle_buffer_length=shuffle_buffer_length,
)
def __init__(
self,
dataset: Dataset,
*,
transform: Transformation,
batch_size: int,
stack_fn: Callable,
# FIXME: the following aren't used
num_workers: Optional[int] = None,
num_prefetch: Optional[int] = None,
shuffle_buffer_length: Optional[int] = None,
) -> None:
self.transformed_dataset = TransformedDataset(
dataset,
transform,
is_train=False,
)
self.batch_size = batch_size
self.stack_fn = stack_fn
def make_evaluation_predictions(
dataset: Dataset, predictor: Predictor, num_eval_samples: int
) -> Tuple[Iterator[Forecast], Iterator[pd.Series]]:
"""
Return predictions on the last portion of predict_length time units of the
target. Such portion is cut before making predictions, such a function can
be used in evaluations where accuracy is evaluated on the last portion of
the target.
Parameters
----------
dataset
Dataset where the evaluation will happen. Only the portion excluding
the prediction_length portion is used when making prediction.
predictor
Model used to draw predictions.
num_eval_samples
Number of samples to draw on the model when evaluating.
Returns
-------
"""
prediction_length = predictor.prediction_length
freq = predictor.freq
def add_ts_dataframe(data_iterator: Iterator[DataEntry]) -> DataEntry:
for data_entry in data_iterator:
data = data_entry.copy()
index = pd.date_range(
start=data["start"],
freq=freq,
periods=data["target"].shape[-1],
)
data["ts"] = pd.DataFrame(
index=index, data=data["target"].transpose()
)
yield data
def ts_iter(dataset: Dataset) -> pd.DataFrame:
for data_entry in add_ts_dataframe(iter(dataset)):
yield data_entry["ts"]
def truncate_target(data):
data = data.copy()
target = data["target"]
assert (
target.shape[-1] >= prediction_length
) # handles multivariate case (target_dim, history_length)
data["target"] = target[..., :-prediction_length]
return data
# TODO filter out time series with target shorter than prediction length
# TODO or fix the evaluator so it supports missing values instead (all
# TODO the test set may be gone otherwise with such a filtering)
dataset_trunc = TransformedDataset(
dataset, transformations=[transform.AdhocTransform(truncate_target)]
)
return (
predictor.predict(dataset_trunc, num_eval_samples=num_eval_samples),
ts_iter(dataset),
)
def TrainDataLoader(
dataset: Dataset,
*,
transform: Transformation,
batch_size: int,
stack_fn: Callable,
num_batches_per_epoch: Optional[int] = None,
num_workers: Optional[int] = None,
num_prefetch: Optional[int] = None,
shuffle_buffer_length: Optional[int] = None,
decode_fn: Callable = lambda x: x,
):
"""Construct an iterator of batches for training purposes.
This function wraps around ``DataLoader`` to offer training-specific
behaviour and options, as follows:
1. The provided dataset is iterated cyclically, so that one can go
over it multiple times in a single epoch.
2. A transformation must be provided, that is lazily applied as the
dataset is being iterated; this is useful e.g. to slice random instances
of fixed length out of each time series in the dataset.
3. The resulting batches can be iterated in a pseudo-shuffled order.
The returned object is a stateful iterator, whose length is either
``num_batches_per_epoch`` (if not ``None``) or infinite (otherwise).
Parameters
----------
dataset
Data to iterate over.
transform
Transformation to be lazily applied as data is being iterated.
The transformation is applied in "training mode" (``is_train=True``).
batch_size
Number of entries to include in a batch.
stack_fn
Function to use to stack data entries into batches.
This can be used to set a specific array type or computing device
the arrays should end up onto (CPU, GPU).
num_batches_per_epoch
Length of the iterator. If ``None``, then the iterator is endless.
num_workers
Number of worker processes to use. Default: None.
num_prefetch
Sets the length of the queue of batches being produced by worker processes.
(Only meaningful when ``num_workers is not None``).
shuffle_buffer_length
Size of the buffer used for shuffling. Default: None, in which case no
shuffling occurs.
decode_fn
A function called on each batch after it's been taken out of the queue.
(Only meaningful when ``num_workers is not None``).
Returns
-------
Iterator[DataBatch]
An iterator of batches.
"""
transformed_dataset = TransformedDataset(
Cyclic(dataset), transform, is_train=True
)
data_iterable = (
PseudoShuffled(
transformed_dataset, shuffle_buffer_length=shuffle_buffer_length
)
if shuffle_buffer_length is not None
else transformed_dataset
)
data_loader = DataLoader(
data_iterable=data_iterable,
batch_size=batch_size,
stack_fn=stack_fn,
num_workers=num_workers,
num_prefetch=num_prefetch,
decode_fn=decode_fn,
)
return (
iter(data_loader)
if num_batches_per_epoch is None
else IterableSlice(iter(data_loader), num_batches_per_epoch)
)
def make_evaluation_predictions(
dataset: Dataset,
predictor: Predictor,
num_samples: int = 100,
) -> Tuple[Iterator[Forecast], Iterator[pd.Series]]:
"""
Returns predictions for the trailing prediction_length observations of the given
time series, using the given predictor.
The predictor will take as input the given time series without the trailing
prediction_length observations.
Parameters
----------
dataset
Dataset where the evaluation will happen. Only the portion excluding
the prediction_length portion is used when making prediction.
predictor
Model used to draw predictions.
num_samples
Number of samples to draw on the model when evaluating. Only sampling-based
models will use this.
Returns
-------
Tuple[Iterator[Forecast], Iterator[pd.Series]]
A pair of iterators, the first one yielding the forecasts, and the second
one yielding the corresponding ground truth series.
"""
prediction_length = predictor.prediction_length
freq = predictor.freq
lead_time = predictor.lead_time
def add_ts_dataframe(
data_iterator: Iterator[DataEntry], ) -> Iterator[DataEntry]:
for data_entry in data_iterator:
data = data_entry.copy()
index = pd.date_range(
start=data["start"],
freq=freq,
periods=data["target"].shape[-1],
)
data["ts"] = pd.DataFrame(index=index,
data=data["target"].transpose())
yield data
def ts_iter(dataset: Dataset) -> pd.DataFrame:
for data_entry in add_ts_dataframe(iter(dataset)):
yield data_entry["ts"]
def truncate_target(data):
data = data.copy()
target = data["target"]
assert (target.shape[-1] >= prediction_length
) # handles multivariate case (target_dim, history_length)
data["target"] = target[..., :-prediction_length - lead_time]
return data
# TODO filter out time series with target shorter than prediction length
# TODO or fix the evaluator so it supports missing values instead (all
# TODO the test set may be gone otherwise with such a filtering)
dataset_trunc = TransformedDataset(
dataset, transformation=transform.AdhocTransform(truncate_target))
return (
predictor.predict(dataset_trunc, num_samples=num_samples),
ts_iter(dataset),
)