def test_distribution(dataloaders_with_covariates, tmp_path, accelerator,
gpus):
if isinstance(gpus, int) and gpus == 0: # only run test on GPU
return
train_dataloader = dataloaders_with_covariates["train"]
val_dataloader = dataloaders_with_covariates["val"]
net = TemporalFusionTransformer.from_dataset(train_dataloader.dataset, )
logger = TensorBoardLogger(tmp_path)
checkpoint = ModelCheckpoint(filepath=tmp_path)
trainer = pl.Trainer(
checkpoint_callback=checkpoint,
max_epochs=3,
gpus=list(range(torch.cuda.device_count())),
weights_summary="top",
gradient_clip_val=0.1,
fast_dev_run=True,
logger=logger,
accelerator=accelerator,
)
try:
trainer.fit(
net,
train_dataloader=train_dataloader,
val_dataloaders=val_dataloader,
)
finally:
shutil.rmtree(tmp_path, ignore_errors=True)
def test_distribution(dataloaders_with_covariates, tmp_path, distributed_backend, gpus):
if gpus == 0: # only run test on GPU
return
train_dataloader = dataloaders_with_covariates["train"]
val_dataloader = dataloaders_with_covariates["val"]
net = TemporalFusionTransformer.from_dataset(
train_dataloader.dataset,
)
logger = TensorBoardLogger(tmp_path)
checkpoint = ModelCheckpoint(filepath=tmp_path)
trainer = pl.Trainer(
checkpoint_callback=checkpoint,
max_epochs=3,
gpus=gpus,
weights_summary="top",
gradient_clip_val=0.1,
fast_dev_run=True,
logger=logger,
distributed_backend=distributed_backend,
)
try:
trainer.fit(
net,
train_dataloader=train_dataloader,
val_dataloaders=val_dataloader,
)
finally:
shutil.rmtree(tmp_path, ignore_errors=True)
def run_train(cfg, trainset, validset) -> None:
import multiprocessing
import pytorch_lightning as pl
from pytorch_lightning.callbacks import EarlyStopping, LearningRateMonitor
from pytorch_lightning.loggers import TensorBoardLogger
from pytorch_forecasting.metrics import QuantileLoss
from pytorch_forecasting.models import TemporalFusionTransformer
# stop training, when loss metric does not improve on validation set
early_stop_callback = EarlyStopping(
monitor="val_loss",
min_delta=1e-4,
patience=10,
verbose=False,
mode="min"
)
lr_monitor = LearningRateMonitor() # log the learning rate
logger = TensorBoardLogger("result/lightning_logs") # log to tensorboard
# create trainer
params = cfg.get("trainer").params
trainer = pl.Trainer(
callbacks=[lr_monitor, early_stop_callback],
logger=logger,
**params,
)
# initialise model
params = cfg.get("model").params
tft = TemporalFusionTransformer.from_dataset(
trainset,
loss=QuantileLoss(),
**params,
)
print(tft.size()) # 29.6k parameters in model
n_cores = multiprocessing.cpu_count()
loader_trainset = trainset.to_dataloader(
train=True, batch_size=cfg.get("trainset").batch_size, num_workers=n_cores
)
loader_validset = validset.to_dataloader(
train=False, batch_size=cfg.get("validset").batch_size, num_workers=n_cores
)
# fit network
trainer.fit(
tft,
train_dataloader=loader_trainset,
val_dataloaders=loader_validset,
)
return
def model(dataloaders_with_covariates):
dataset = dataloaders_with_covariates["train"].dataset
net = TemporalFusionTransformer.from_dataset(
dataset,
learning_rate=0.15,
hidden_size=4,
attention_head_size=1,
dropout=0.2,
hidden_continuous_size=2,
loss=PoissonLoss(),
output_size=1,
log_interval=5,
log_val_interval=1,
log_gradient_flow=True,
)
return net
def test_prediction_with_dataloder_raw(data_with_covariates, tmp_path):
# tests correct concatenation of raw output
test_data = data_with_covariates.copy()
np.random.seed(2)
test_data = test_data.sample(frac=0.5)
dataset = TimeSeriesDataSet(
test_data,
time_idx="time_idx",
max_encoder_length=8,
max_prediction_length=10,
min_prediction_length=1,
min_encoder_length=1,
target="volume",
group_ids=["agency", "sku"],
constant_fill_strategy=dict(volume=0.0),
allow_missing_timesteps=True,
time_varying_unknown_reals=["volume"],
time_varying_known_reals=["time_idx"],
target_normalizer=GroupNormalizer(groups=["agency", "sku"]),
)
net = TemporalFusionTransformer.from_dataset(
dataset,
learning_rate=1e-6,
hidden_size=4,
attention_head_size=1,
dropout=0.2,
hidden_continuous_size=2,
log_interval=1,
log_val_interval=1,
log_gradient_flow=True,
)
logger = TensorBoardLogger(tmp_path)
trainer = pl.Trainer(max_epochs=1, gradient_clip_val=1e-6, logger=logger)
trainer.fit(net,
train_dataloaders=dataset.to_dataloader(batch_size=4,
num_workers=0))
# choose small batch size to provoke issue
res = net.predict(dataset.to_dataloader(batch_size=2, num_workers=0),
mode="raw")
# check that interpretation works
net.interpret_output(res)["attention"]
assert net.interpret_output(res.iget(
slice(1)))["attention"].size() == torch.Size(
(1, net.hparams.max_encoder_length))
def model(dataloaders_with_covariates, gpus):
dataset = dataloaders_with_covariates["train"].dataset
net = TemporalFusionTransformer.from_dataset(
dataset,
learning_rate=0.15,
hidden_size=4,
attention_head_size=1,
dropout=0.2,
hidden_continuous_size=2,
loss=PoissonLoss(),
output_size=1,
log_interval=5,
log_val_interval=1,
log_gradient_flow=True,
)
if isinstance(gpus, list) and len(gpus) > 0: # only run test on GPU
net.to(gpus[0])
return net
def test_integration(multiple_dataloaders_with_covariates, tmp_path, gpus):
train_dataloader = multiple_dataloaders_with_covariates["train"]
val_dataloader = multiple_dataloaders_with_covariates["val"]
early_stop_callback = EarlyStopping(monitor="val_loss",
min_delta=1e-4,
patience=1,
verbose=False,
mode="min")
# check training
logger = TensorBoardLogger(tmp_path)
checkpoint = ModelCheckpoint(filepath=tmp_path)
trainer = pl.Trainer(
checkpoint_callback=checkpoint,
max_epochs=3,
gpus=gpus,
weights_summary="top",
gradient_clip_val=0.1,
callbacks=[early_stop_callback],
fast_dev_run=True,
logger=logger,
)
# test monotone constraints automatically
if "discount_in_percent" in train_dataloader.dataset.reals:
monotone_constaints = {"discount_in_percent": +1}
cuda_context = torch.backends.cudnn.flags(enabled=False)
else:
monotone_constaints = {}
cuda_context = nullcontext()
with cuda_context:
if isinstance(train_dataloader.dataset.target_normalizer,
NaNLabelEncoder):
loss = CrossEntropy()
elif isinstance(train_dataloader.dataset.target_normalizer,
MultiNormalizer):
loss = MultiLoss([
CrossEntropy()
if isinstance(normalizer, NaNLabelEncoder) else QuantileLoss()
for normalizer in
train_dataloader.dataset.target_normalizer.normalizers
])
else:
loss = QuantileLoss()
net = TemporalFusionTransformer.from_dataset(
train_dataloader.dataset,
learning_rate=0.15,
hidden_size=4,
attention_head_size=1,
dropout=0.2,
hidden_continuous_size=2,
loss=loss,
log_interval=5,
log_val_interval=1,
log_gradient_flow=True,
monotone_constaints=monotone_constaints,
)
net.size()
try:
trainer.fit(
net,
train_dataloader=train_dataloader,
val_dataloaders=val_dataloader,
)
# check loading
net = TemporalFusionTransformer.load_from_checkpoint(
checkpoint.best_model_path)
# check prediction
net.predict(val_dataloader,
fast_dev_run=True,
return_index=True,
return_decoder_lengths=True)
# check prediction on gpu
if not (isinstance(gpus, int) and gpus == 0):
net.to("cuda")
net.predict(val_dataloader,
fast_dev_run=True,
return_index=True,
return_decoder_lengths=True)
finally:
shutil.rmtree(tmp_path, ignore_errors=True)
def test_integration(multiple_dataloaders_with_covariates, tmp_path, gpus):
train_dataloader = multiple_dataloaders_with_covariates["train"]
val_dataloader = multiple_dataloaders_with_covariates["val"]
early_stop_callback = EarlyStopping(monitor="val_loss", min_delta=1e-4, patience=1, verbose=False, mode="min")
# check training
logger = TensorBoardLogger(tmp_path)
trainer = pl.Trainer(
max_epochs=2,
gpus=gpus,
weights_summary="top",
gradient_clip_val=0.1,
callbacks=[early_stop_callback],
checkpoint_callback=True,
default_root_dir=tmp_path,
limit_train_batches=2,
limit_val_batches=2,
logger=logger,
)
# test monotone constraints automatically
if "discount_in_percent" in train_dataloader.dataset.reals:
monotone_constaints = {"discount_in_percent": +1}
cuda_context = torch.backends.cudnn.flags(enabled=False)
else:
monotone_constaints = {}
cuda_context = nullcontext()
with cuda_context:
if isinstance(train_dataloader.dataset.target_normalizer, NaNLabelEncoder):
loss = CrossEntropy()
elif isinstance(train_dataloader.dataset.target_normalizer, MultiNormalizer):
loss = MultiLoss(
[
CrossEntropy() if isinstance(normalizer, NaNLabelEncoder) else QuantileLoss()
for normalizer in train_dataloader.dataset.target_normalizer.normalizers
]
)
else:
loss = QuantileLoss()
net = TemporalFusionTransformer.from_dataset(
train_dataloader.dataset,
learning_rate=0.15,
hidden_size=4,
attention_head_size=1,
dropout=0.2,
hidden_continuous_size=2,
loss=loss,
log_interval=5,
log_val_interval=1,
log_gradient_flow=True,
monotone_constaints=monotone_constaints,
)
net.size()
try:
trainer.fit(
net,
train_dataloader=train_dataloader,
val_dataloaders=val_dataloader,
)
# check loading
net = TemporalFusionTransformer.load_from_checkpoint(trainer.checkpoint_callback.best_model_path)
# check prediction
predictions, x, index = net.predict(val_dataloader, return_index=True, return_x=True)
pred_len = len(multiple_dataloaders_with_covariates["val"].dataset)
# check that output is of correct shape
def check(x):
if isinstance(x, (tuple, list)):
for xi in x:
check(xi)
elif isinstance(x, dict):
for xi in x.values():
check(xi)
else:
assert pred_len == x.shape[0], "first dimension should be prediction length"
check(predictions)
check(x)
check(index)
# check prediction on gpu
if not (isinstance(gpus, int) and gpus == 0):
net.to("cuda")
net.predict(val_dataloader, fast_dev_run=True, return_index=True, return_decoder_lengths=True)
finally:
shutil.rmtree(tmp_path, ignore_errors=True)
def test_init_shared_network(dataloaders_with_covariates):
dataset = dataloaders_with_covariates["train"].dataset
net = TemporalFusionTransformer.from_dataset(dataset, share_single_variable_networks=True)
net.predict(dataset)
max_epochs=30,
gpus=0, # train on CPU, use gpus = [0] to run on GPU
gradient_clip_val=0.1,
early_stop_callback=early_stop_callback,
limit_train_batches=30, # running validation every 30 batches
# fast_dev_run=True, # comment in to quickly check for bugs
callbacks=[lr_logger],
logger=logger,
)
# initialise model
tft = TemporalFusionTransformer.from_dataset(
training,
learning_rate=0.03,
hidden_size=16, # biggest influence network size
attention_head_size=1,
dropout=0.1,
hidden_continuous_size=8,
output_size=7, # QuantileLoss has 7 quantiles by default
loss=QuantileLoss(),
log_interval=10, # log example every 10 batches
reduce_on_plateau_patience=4, # reduce learning automatically
)
tft.size() # 29.6k parameters in model
# fit network
trainer.fit(tft,
train_dataloader=train_dataloader,
val_dataloaders=val_dataloader)
#%%
"""
Evaluating the trained model
"""
max_epochs=70,
gpus=[0],
weights_summary="top",
gradient_clip_val=0.07491732457954926,
limit_train_batches=
30, # comment in for training, running valiation every 30 batches
# fast_dev_run=True, # comment in to check that networkor dataset has no serious bugs
callbacks=[early_stop_callback])
# initialise model
tft = TemporalFusionTransformer.from_dataset(
training,
hidden_size=20,
hidden_continuous_size=9,
attention_head_size=4,
learning_rate=0.00417,
dropout=0.12522982138188382,
reduce_on_plateau_patience=4,
log_interval=100, # log example every 10 batches
loss=QuantileLoss(),
output_size=7, # QuantileLoss has 7 quantiles by default
)
print(f"Number of parameters in network: {tft.size() / 1e3:.1f}k")
# # find optimal learning rate
# res = trainer.lr_find(
# tft,
# train_dataloader=train_dataloader,
# val_dataloaders=val_dataloader,
# max_lr=10.0,
# min_lr=1e-6,
# )