def run_train( forecaster: Estimator, train_dataset: Dataset, validation_dataset: Optional[Dataset], ) -> Predictor: log.metric("train_dataset_stats", train_dataset.calc_stats()) return forecaster.train(train_dataset, validation_dataset)
def run_train(self, dataset) -> Tuple[Estimator, Predictor]: # print out stats log.metric('train_dataset_stats', dataset.calc_stats()) estimator = self.from_hyperparameters(**self.hyperparameters) log.metric('estimator', estimator) return estimator, estimator.train(dataset)
def run_test(self, dataset, estimator, predictor): test_dataset = TransformedDataset( dataset, transformations=[ FilterTransformation(lambda el: el['target'].shape[-1] > predictor.prediction_length) ], ) len_orig = len(dataset) len_filtered = len(test_dataset) if len_orig > len_filtered: logging.warning( 'Not all time-series in the test-channel have ' 'enough data to be used for evaluation. Proceeding with ' f'{len_filtered}/{len_orig} ' f'(~{int(len_filtered/len_orig*100)}%) items.') try: log.metric('test_dataset_stats', test_dataset.calc_stats()) except GluonTSDataError as error: logging.error( f"Failure whilst calculating stats for test dataset: {error}") return if isinstance(estimator, GluonEstimator) and isinstance( predictor, GluonPredictor): inference_data_loader = InferenceDataLoader( dataset=test_dataset, transform=predictor.input_transform, batch_size=estimator.trainer.batch_size, ctx=estimator.trainer.ctx, float_type=estimator.float_type, ) if estimator.trainer.hybridize: predictor.hybridize(batch=next(iter(inference_data_loader))) if self.hyperparameters.get('use_symbol_block_predictor'): predictor = predictor.as_symbol_block_predictor( batch=next(iter(inference_data_loader))) num_eval_samples = self.hyperparameters.get('num_eval_samples', 100) quantiles = self.hyperparameters.get( 'quantiles', (0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9)) # we only log aggregate metrics for now as item metrics may be # very large predictions, input_timeseries = backtest.make_evaluation_predictions( test_dataset, predictor, num_eval_samples) agg_metrics, _item_metrics = Evaluator(quantiles=quantiles)( input_timeseries, predictions, num_series=len_filtered) log.metric("agg_metrics", agg_metrics)
def run_test(forecaster, test_dataset): agg_metrics, _item_metrics = backtest.backtest_metrics( train_dataset=None, test_dataset=test_dataset, forecaster=forecaster, evaluator=Evaluator(quantiles=(0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9)), num_eval_samples=100, ) # we only log aggregate metrics for now as item metrics may be # very large log.metric("agg_metrics", agg_metrics)
def run_train(forecaster, train_dataset) -> Predictor: assert isinstance(forecaster, Estimator) log.metric('train_dataset_stats', train_dataset.calc_stats()) return forecaster.train(train_dataset)