def test_BucketInstanceSampler():
N = 6
train_length = 2
pred_length = 1
ds = make_dataset(N, train_length)
dataset_stats = calculate_dataset_statistics(ds)
t = transform.Chain(
trans=[
transform.InstanceSplitter(
target_field=FieldName.TARGET,
is_pad_field=FieldName.IS_PAD,
start_field=FieldName.START,
forecast_start_field=FieldName.FORECAST_START,
train_sampler=transform.BucketInstanceSampler(
dataset_stats.scale_histogram
),
past_length=train_length,
future_length=pred_length,
pick_incomplete=True,
)
]
)
scale_hist = ScaleHistogram()
repetition = 200
for i in range(repetition):
for data in t(iter(ds), is_train=True):
target_values = data["past_target"]
# for simplicity, discard values that are zeros to avoid confusion with padding
target_values = target_values[target_values > 0]
scale_hist.add(target_values)
expected_values = {i: repetition for i in range(1, N)}
found_values = scale_hist.bin_counts
for i in range(1, N):
assert abs(
expected_values[i] - found_values[i] < expected_values[i] * 0.3
)
def backtest_metrics(
train_dataset: Optional[Dataset],
test_dataset: Dataset,
forecaster: Union[Estimator, Predictor],
evaluator=Evaluator(quantiles=(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9)),
num_samples: int = 100,
logging_file: Optional[str] = None,
):
"""
Parameters
----------
train_dataset
Dataset to use for training.
test_dataset
Dataset to use for testing.
forecaster
An estimator or a predictor to use for generating predictions.
evaluator
Evaluator to use.
num_samples
Number of samples to use when generating sample-based forecasts.
logging_file
If specified, information of the backtest is redirected to this file.
Returns
-------
tuple
A tuple of aggregate metrics and per-time-series metrics obtained by
training `forecaster` on `train_dataset` and evaluating the resulting
`evaluator` provided on the `test_dataset`.
"""
if logging_file is not None:
log_formatter = logging.Formatter(
"[%(asctime)s %(levelname)s %(thread)d] %(message)s",
datefmt="%m/%d/%Y %H:%M:%S",
)
logger = logging.getLogger(__name__)
handler = logging.FileHandler(logging_file)
handler.setFormatter(log_formatter)
logger.addHandler(handler)
else:
logger = logging.getLogger(__name__)
if train_dataset is not None:
train_statistics = calculate_dataset_statistics(train_dataset)
serialize_message(logger, train_dataset_stats_key, train_statistics)
test_statistics = calculate_dataset_statistics(test_dataset)
serialize_message(logger, test_dataset_stats_key, test_statistics)
if isinstance(forecaster, Estimator):
serialize_message(logger, estimator_key, forecaster)
assert train_dataset is not None
predictor = forecaster.train(train_dataset)
else:
predictor = forecaster
forecast_it, ts_it = make_evaluation_predictions(
test_dataset, predictor=predictor, num_samples=num_samples
)
agg_metrics, item_metrics = evaluator(
ts_it, forecast_it, num_series=len(test_dataset)
)
# we only log aggregate metrics for now as item metrics may be very large
for name, value in agg_metrics.items():
serialize_message(logger, f"metric-{name}", value)
if logging_file is not None:
# Close the file handler to avoid letting the file open.
# https://stackoverflow.com/questions/24816456/python-logging-wont-shutdown
logger.removeHandler(handler)
del logger, handler
return agg_metrics, item_metrics