def _integration(
data_with_covariates, tmp_path, gpus, cell_type="LSTM", data_loader_kwargs={}, clip_target: bool = False, **kwargs
):
if clip_target:
data_with_covariates["target"] = data_with_covariates["volume"].clip(1e-3, 1.0)
else:
data_with_covariates["target"] = data_with_covariates["volume"]
data_loader_default_kwargs = dict(
target="target",
time_varying_known_reals=["price_actual"],
time_varying_unknown_reals=["target"],
static_categoricals=["agency"],
add_relative_time_idx=True,
)
data_loader_default_kwargs.update(data_loader_kwargs)
dataloaders_with_covariates = make_dataloaders(data_with_covariates, **data_loader_default_kwargs)
train_dataloader = dataloaders_with_covariates["train"]
val_dataloader = dataloaders_with_covariates["val"]
early_stop_callback = EarlyStopping(monitor="val_loss", min_delta=1e-4, patience=1, verbose=False, mode="min")
logger = TensorBoardLogger(tmp_path)
trainer = pl.Trainer(
max_epochs=3,
gpus=gpus,
weights_summary="top",
gradient_clip_val=0.1,
callbacks=[early_stop_callback],
checkpoint_callback=True,
default_root_dir=tmp_path,
limit_train_batches=2,
limit_val_batches=2,
logger=logger,
)
net = DeepAR.from_dataset(
train_dataloader.dataset,
cell_type=cell_type,
learning_rate=0.15,
log_gradient_flow=True,
log_interval=1000,
n_plotting_samples=100,
**kwargs,
)
net.size()
try:
trainer.fit(
net,
train_dataloader=train_dataloader,
val_dataloaders=val_dataloader,
)
# check loading
net = DeepAR.load_from_checkpoint(trainer.checkpoint_callback.best_model_path)
# check prediction
net.predict(val_dataloader, fast_dev_run=True, return_index=True, return_decoder_lengths=True)
finally:
shutil.rmtree(tmp_path, ignore_errors=True)
net.predict(val_dataloader, fast_dev_run=True, return_index=True, return_decoder_lengths=True)
def test_integration_for_multiple_targets(data_with_covariates, tmp_path, gpus):
_integration(
make_dataloaders(
data_with_covariates,
time_varying_unknown_reals=["volume", "discount"],
target=["volume", "discount"],
),
tmp_path,
gpus,
)
def test_distribution_loss(data_with_covariates, tmp_path, gpus):
data_with_covariates = data_with_covariates.assign(
volume=lambda x: x.volume.round())
dataloaders_with_covariates = make_dataloaders(
data_with_covariates,
target="volume",
time_varying_known_reals=["price_actual"],
time_varying_unknown_reals=["volume"],
static_categoricals=["agency"],
add_relative_time_idx=True,
target_normalizer=GroupNormalizer(groups=["agency", "sku"],
center=False),
)
_integration(dataloaders_with_covariates,
tmp_path,
gpus,
loss=NegativeBinomialDistributionLoss())
def test_mqf2_loss(data_with_covariates, tmp_path, gpus):
data_with_covariates = data_with_covariates.assign(
volume=lambda x: x.volume.round())
dataloaders_with_covariates = make_dataloaders(
data_with_covariates,
target="volume",
time_varying_known_reals=["price_actual"],
time_varying_unknown_reals=["volume"],
static_categoricals=["agency"],
add_relative_time_idx=True,
target_normalizer=GroupNormalizer(groups=["agency", "sku"],
center=False,
transformation="softplus"),
)
prediction_length = dataloaders_with_covariates[
"train"].dataset.min_prediction_length
_integration(
dataloaders_with_covariates,
tmp_path,
gpus,
loss=MQF2DistributionLoss(prediction_length=prediction_length))
def _integration(data_with_covariates,
tmp_path,
gpus,
data_loader_kwargs={},
train_only=False,
**kwargs):
data_loader_default_kwargs = dict(
target="target",
time_varying_known_reals=["price_actual"],
time_varying_unknown_reals=["target"],
static_categoricals=["agency"],
add_relative_time_idx=True,
)
data_loader_default_kwargs.update(data_loader_kwargs)
dataloaders_with_covariates = make_dataloaders(
data_with_covariates, **data_loader_default_kwargs)
train_dataloader = dataloaders_with_covariates["train"]
val_dataloader = dataloaders_with_covariates["val"]
test_dataloader = dataloaders_with_covariates["test"]
early_stop_callback = EarlyStopping(monitor="val_loss",
min_delta=1e-4,
patience=1,
verbose=False,
mode="min",
strict=False)
logger = TensorBoardLogger(tmp_path)
trainer = pl.Trainer(
max_epochs=3,
gpus=gpus,
gradient_clip_val=0.1,
callbacks=[early_stop_callback],
enable_checkpointing=True,
default_root_dir=tmp_path,
limit_train_batches=2,
limit_val_batches=2,
limit_test_batches=2,
logger=logger,
)
net = DecoderMLP.from_dataset(train_dataloader.dataset,
learning_rate=0.015,
log_gradient_flow=True,
log_interval=1000,
hidden_size=10,
**kwargs)
net.size()
try:
if train_only:
trainer.fit(net, train_dataloaders=train_dataloader)
else:
trainer.fit(
net,
train_dataloaders=train_dataloader,
val_dataloaders=val_dataloader,
)
# check loading
net = DecoderMLP.load_from_checkpoint(
trainer.checkpoint_callback.best_model_path)
# check prediction
net.predict(val_dataloader,
fast_dev_run=True,
return_index=True,
return_decoder_lengths=True)
# check test dataloader
test_outputs = trainer.test(net, dataloaders=test_dataloader)
assert len(test_outputs) > 0
finally:
shutil.rmtree(tmp_path, ignore_errors=True)
net.predict(val_dataloader,
fast_dev_run=True,
return_index=True,
return_decoder_lengths=True)