def run_train(
forecaster: Estimator,
train_dataset: Dataset,
validation_dataset: Optional[Dataset],
) -> Predictor:
log.metric("train_dataset_stats", train_dataset.calc_stats())
return forecaster.train(train_dataset, validation_dataset)
def run_train(self, dataset) -> Tuple[Estimator, Predictor]:
# print out stats
log.metric('train_dataset_stats', dataset.calc_stats())
estimator = self.from_hyperparameters(**self.hyperparameters)
log.metric('estimator', estimator)
return estimator, estimator.train(dataset)
def run_test(self, dataset, estimator, predictor):
test_dataset = TransformedDataset(
dataset,
transformations=[
FilterTransformation(lambda el: el['target'].shape[-1] >
predictor.prediction_length)
],
)
len_orig = len(dataset)
len_filtered = len(test_dataset)
if len_orig > len_filtered:
logging.warning(
'Not all time-series in the test-channel have '
'enough data to be used for evaluation. Proceeding with '
f'{len_filtered}/{len_orig} '
f'(~{int(len_filtered/len_orig*100)}%) items.')
try:
log.metric('test_dataset_stats', test_dataset.calc_stats())
except GluonTSDataError as error:
logging.error(
f"Failure whilst calculating stats for test dataset: {error}")
return
if isinstance(estimator, GluonEstimator) and isinstance(
predictor, GluonPredictor):
inference_data_loader = InferenceDataLoader(
dataset=test_dataset,
transform=predictor.input_transform,
batch_size=estimator.trainer.batch_size,
ctx=estimator.trainer.ctx,
float_type=estimator.float_type,
)
if estimator.trainer.hybridize:
predictor.hybridize(batch=next(iter(inference_data_loader)))
if self.hyperparameters.get('use_symbol_block_predictor'):
predictor = predictor.as_symbol_block_predictor(
batch=next(iter(inference_data_loader)))
num_eval_samples = self.hyperparameters.get('num_eval_samples', 100)
quantiles = self.hyperparameters.get(
'quantiles', (0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9))
# we only log aggregate metrics for now as item metrics may be
# very large
predictions, input_timeseries = backtest.make_evaluation_predictions(
test_dataset, predictor, num_eval_samples)
agg_metrics, _item_metrics = Evaluator(quantiles=quantiles)(
input_timeseries, predictions, num_series=len_filtered)
log.metric("agg_metrics", agg_metrics)
def run_test(forecaster, test_dataset):
agg_metrics, _item_metrics = backtest.backtest_metrics(
train_dataset=None,
test_dataset=test_dataset,
forecaster=forecaster,
evaluator=Evaluator(quantiles=(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8,
0.9)),
num_eval_samples=100,
)
# we only log aggregate metrics for now as item metrics may be
# very large
log.metric("agg_metrics", agg_metrics)
def run_train(forecaster, train_dataset) -> Predictor:
assert isinstance(forecaster, Estimator)
log.metric('train_dataset_stats', train_dataset.calc_stats())
return forecaster.train(train_dataset)
