def test_integration(multiple_dataloaders_with_covariates, tmp_path, gpus): train_dataloader = multiple_dataloaders_with_covariates["train"] val_dataloader = multiple_dataloaders_with_covariates["val"] early_stop_callback = EarlyStopping(monitor="val_loss", min_delta=1e-4, patience=1, verbose=False, mode="min") # check training logger = TensorBoardLogger(tmp_path) trainer = pl.Trainer( max_epochs=2, gpus=gpus, weights_summary="top", gradient_clip_val=0.1, callbacks=[early_stop_callback], checkpoint_callback=True, default_root_dir=tmp_path, limit_train_batches=2, limit_val_batches=2, logger=logger, ) # test monotone constraints automatically if "discount_in_percent" in train_dataloader.dataset.reals: monotone_constaints = {"discount_in_percent": +1} cuda_context = torch.backends.cudnn.flags(enabled=False) else: monotone_constaints = {} cuda_context = nullcontext() with cuda_context: if isinstance(train_dataloader.dataset.target_normalizer, NaNLabelEncoder): loss = CrossEntropy() elif isinstance(train_dataloader.dataset.target_normalizer, MultiNormalizer): loss = MultiLoss( [ CrossEntropy() if isinstance(normalizer, NaNLabelEncoder) else QuantileLoss() for normalizer in train_dataloader.dataset.target_normalizer.normalizers ] ) else: loss = QuantileLoss() net = TemporalFusionTransformer.from_dataset( train_dataloader.dataset, learning_rate=0.15, hidden_size=4, attention_head_size=1, dropout=0.2, hidden_continuous_size=2, loss=loss, log_interval=5, log_val_interval=1, log_gradient_flow=True, monotone_constaints=monotone_constaints, ) net.size() try: trainer.fit( net, train_dataloader=train_dataloader, val_dataloaders=val_dataloader, ) # check loading net = TemporalFusionTransformer.load_from_checkpoint(trainer.checkpoint_callback.best_model_path) # check prediction predictions, x, index = net.predict(val_dataloader, return_index=True, return_x=True) pred_len = len(multiple_dataloaders_with_covariates["val"].dataset) # check that output is of correct shape def check(x): if isinstance(x, (tuple, list)): for xi in x: check(xi) elif isinstance(x, dict): for xi in x.values(): check(xi) else: assert pred_len == x.shape[0], "first dimension should be prediction length" check(predictions) check(x) check(index) # check prediction on gpu if not (isinstance(gpus, int) and gpus == 0): net.to("cuda") net.predict(val_dataloader, fast_dev_run=True, return_index=True, return_decoder_lengths=True) finally: shutil.rmtree(tmp_path, ignore_errors=True)
def test_integration(multiple_dataloaders_with_covariates, tmp_path, gpus): train_dataloader = multiple_dataloaders_with_covariates["train"] val_dataloader = multiple_dataloaders_with_covariates["val"] early_stop_callback = EarlyStopping(monitor="val_loss", min_delta=1e-4, patience=1, verbose=False, mode="min") # check training logger = TensorBoardLogger(tmp_path) checkpoint = ModelCheckpoint(filepath=tmp_path) trainer = pl.Trainer( checkpoint_callback=checkpoint, max_epochs=3, gpus=gpus, weights_summary="top", gradient_clip_val=0.1, callbacks=[early_stop_callback], fast_dev_run=True, logger=logger, ) # test monotone constraints automatically if "discount_in_percent" in train_dataloader.dataset.reals: monotone_constaints = {"discount_in_percent": +1} cuda_context = torch.backends.cudnn.flags(enabled=False) else: monotone_constaints = {} cuda_context = nullcontext() with cuda_context: if isinstance(train_dataloader.dataset.target_normalizer, NaNLabelEncoder): loss = CrossEntropy() elif isinstance(train_dataloader.dataset.target_normalizer, MultiNormalizer): loss = MultiLoss([ CrossEntropy() if isinstance(normalizer, NaNLabelEncoder) else QuantileLoss() for normalizer in train_dataloader.dataset.target_normalizer.normalizers ]) else: loss = QuantileLoss() net = TemporalFusionTransformer.from_dataset( train_dataloader.dataset, learning_rate=0.15, hidden_size=4, attention_head_size=1, dropout=0.2, hidden_continuous_size=2, loss=loss, log_interval=5, log_val_interval=1, log_gradient_flow=True, monotone_constaints=monotone_constaints, ) net.size() try: trainer.fit( net, train_dataloader=train_dataloader, val_dataloaders=val_dataloader, ) # check loading net = TemporalFusionTransformer.load_from_checkpoint( checkpoint.best_model_path) # check prediction net.predict(val_dataloader, fast_dev_run=True, return_index=True, return_decoder_lengths=True) # check prediction on gpu if not (isinstance(gpus, int) and gpus == 0): net.to("cuda") net.predict(val_dataloader, fast_dev_run=True, return_index=True, return_decoder_lengths=True) finally: shutil.rmtree(tmp_path, ignore_errors=True)
trainer.fit(tft, train_dataloader=train_dataloader, val_dataloaders=val_dataloader) #%% """ Evaluating the trained model """ from pytorch_forecasting.metrics import MAE import torch # load the best model according to the validation loss (given that # we use early stopping, this is not necessarily the last epoch) best_model_path = trainer.checkpoint_callback.best_model_path best_tft = TemporalFusionTransformer.load_from_checkpoint(best_model_path) # calculate mean absolute error on validation set actuals = torch.cat([y for x, y in iter(val_dataloader)]) predictions = best_tft.predict(val_dataloader) MAE(predictions, actuals) from pytorch_forecasting.metrics import SMAPE raw_predictions = best_tft.predict(val_dataloader, mode="raw") # calculate metric by which to display predictions, x = best_tft.predict(val_dataloader, return_x=True) mean_losses = SMAPE(reduction="none")(predictions, actuals).mean(1) indices = mean_losses.argsort(descending=True) # sort losses # show only two examples for demonstration purposes for idx in range(2): best_tft.plot_prediction(x,