def test_cross_entropy_loss():
"""
Test masking for cross entropy loss.
"""
set_default_backend("pytorch")
y_pred = torch.rand(10, 10, 10)
y = torch.ones(10, 1, 10)
bins = np.linspace(0, 1, 11)
y[:, 0, :] = 0.55
loss = CrossEntropyLoss(bins, mask=-1.0)
ref = -y_pred[:, 5, :] + torch.log(torch.exp(y_pred).sum(1))
assert np.all(
np.isclose(
loss(y_pred, y).detach().numpy(),
ref.mean().detach().numpy()))
y[5:, :, :] = -1.0
y[:, :, 5:] = -1.0
ref = -y_pred[:5, 5, :5] + torch.log(torch.exp(y_pred[:5, :, :5]).sum(1))
assert np.all(
np.isclose(
loss(y_pred, y).detach().numpy(),
ref.mean().detach().numpy()))
def test_quantile_loss():
"""
Ensure that quantile loss corresponds to half of absolute error
loss and that masking works as expected.
"""
set_default_backend("pytorch")
loss = QuantileLoss([0.5], mask=-1e3)
y_pred = torch.rand(10, 1, 10)
y = torch.rand(10, 1, 10)
l = loss(y_pred, y).detach().numpy()
dy = (y_pred - y).detach().numpy()
l_ref = 0.5 * np.mean(np.abs(dy))
assert np.isclose(l, l_ref)
y_pred = torch.rand(20, 1, 10)
y_pred[10:] = -2e3
y = torch.rand(20, 1, 10)
y[10:] = -2e3
loss = QuantileLoss([0.5], mask=-1e3)
l = loss(y_pred, y).detach().numpy()
l_ref = loss(y_pred[:10], y[:10]).detach().numpy()
assert np.isclose(l, l_ref)
def test_qrnn_training_state():
"""
Ensure that training attributes of models are conserved through
training.
"""
set_default_backend("pytorch")
x = np.random.rand(1024, 16)
y = np.random.rand(1024)
training_data = torch.utils.data.TensorDataset(torch.tensor(x),
torch.tensor(y))
training_loader = torch.utils.data.DataLoader(training_data,
batch_size=128)
model = nn.Sequential(nn.BatchNorm1d(16), nn.Linear(16, 10))
qrnn = QRNN(np.linspace(0.05, 0.95, 10), model=model)
qrnn.model.train(False)
qrnn.train(training_loader, n_epochs=1)
mean = model[0].running_mean.detach().numpy()
assert np.all(np.isclose(mean, 0.0))
var = model[0].running_var.detach().numpy()
assert np.all(np.isclose(var, 1.0))
qrnn.model.train(True)
qrnn.train(training_loader, n_epochs=1)
mean = model[0].running_mean.detach().numpy()
assert not np.all(np.isclose(mean, 0.0))
var = model[0].running_var.detach().numpy()
assert not np.all(np.isclose(var, 1.0))
def test_training_multiple_outputs():
"""
Ensure that training with batch objects as dicts and provided keys
argument works.
"""
set_default_backend("keras")
class MultipleOutputModel(keras.Model):
def __init__(self):
super().__init__()
self.hidden = keras.layers.Dense(128, "relu", input_shape=(16, ))
self.head_1 = keras.layers.Dense(11, None)
self.head_2 = keras.layers.Dense(11, None)
def call(self, x):
x = self.hidden(x)
y_1 = self.head_1(x)
y_2 = self.head_2(x)
return {"y_1": y_1, "y_2": y_2}
x = np.random.rand(1024, 16)
y = np.random.rand(1024)
batched_data = [{
"x": x[i * 128:(i + 1) * 128],
"y": {
"y_1": y[i * 128:(i + 1) * 128],
"y_2": y[i * 128:(i + 1) * 128]
}
} for i in range(1024 // 128)]
model = MultipleOutputModel()
qrnn = QRNN(np.linspace(0.05, 0.95, 11), model=model)
qrnn.train(batched_data, n_epochs=10, keys=("x", "y"))
def test_qrnn(self, backend):
"""
Test training of QRNNs using numpy arrays as input.
"""
set_default_backend(backend)
qrnn = QRNN(np.linspace(0.05, 0.95, 10),
n_inputs=self.x_train.shape[1])
qrnn.train((self.x_train, self.y_train),
validation_data=(self.x_train, self.y_train),
n_epochs=2)
qrnn.predict(self.x_train)
x, qs = qrnn.cdf(self.x_train[:2, :])
assert qs[0] == 0.0
assert qs[-1] == 1.0
x, y = qrnn.pdf(self.x_train[:2, :])
assert x.shape == y.shape
mu = qrnn.posterior_mean(self.x_train[:2, :])
assert len(mu.shape) == 1
r = qrnn.sample_posterior(self.x_train[:4, :], n_samples=2)
assert r.shape == (4, 2)
r = qrnn.sample_posterior_gaussian_fit(self.x_train[:4, :],
n_samples=2)
assert r.shape == (4, 2)
def test_training_multiple_outputs():
"""
Ensure that training with batch objects as dicts and provided keys
argument works.
"""
set_default_backend("pytorch")
class MultipleOutputModel(nn.Module):
def __init__(self):
super().__init__()
self.hidden = nn.Linear(16, 128)
self.head_1 = nn.Linear(128, 11)
self.head_2 = nn.Linear(128, 11)
def forward(self, x):
x = torch.relu(self.hidden(x))
y_1 = self.head_1(x)
y_2 = self.head_2(x)
return {"y_1": y_1, "y_2": y_2}
x = np.random.rand(1024, 16)
y = np.random.rand(1024)
batched_data = [{
"x": torch.tensor(x[i * 128:(i + 1) * 128]),
"y": {
"y_1": torch.tensor(y[i * 128:(i + 1) * 128]),
"y_2": torch.tensor(y[i * 128:(i + 1) * 128])
}
} for i in range(1024 // 128)]
model = MultipleOutputModel()
qrnn = QRNN(np.linspace(0.05, 0.95, 11), model=model)
qrnn.train(batched_data, n_epochs=5, keys=("x", "y"))
def test_drnn_datasets(self, backend):
"""
Provide data as dataset object instead of numpy arrays.
"""
set_default_backend(backend)
backend = get_default_backend()
data = backend.BatchedDataset((self.x_train, self.y_train), 256)
drnn = DRNN(self.bins, n_inputs=self.x_train.shape[1])
drnn.train(data, n_epochs=2)
def test_qrnn_datasets(self, backend):
"""
Provide data as dataset object instead of numpy arrays.
"""
set_default_backend(backend)
backend = get_default_backend()
data = backend.BatchedDataset((self.x_train, self.y_train), 256)
qrnn = QRNN(np.linspace(0.05, 0.95, 10),
n_inputs=self.x_train.shape[1])
qrnn.train(data, n_epochs=2)
def test_training_transformation_mrnn_quantiles():
"""
Ensure that training in transformed space works.
"""
set_default_backend("pytorch")
class MultipleOutputModel(nn.Module):
def __init__(self):
super().__init__()
self.hidden = nn.Linear(16, 128)
self.head_1 = nn.Linear(128, 10)
self.head_2 = nn.Linear(128, 1)
self.head_3 = nn.Linear(128, 10)
def forward(self, x):
x = torch.relu(self.hidden(x))
y_1 = self.head_1(x)
y_2 = self.head_2(x)
y_3 = self.head_3(x)
return {"y_1": y_1, "y_2": y_2, "y_3": y_3}
x = np.random.rand(2024, 16) + 1.0
y = np.sum(x, axis=-1)
y += np.random.normal(size=y.size)
batched_data = [{
"x":
torch.tensor(x[i * 128:(i + 1) * 128], dtype=torch.float32),
"y": {
"y_1": torch.tensor(y[i * 128:(i + 1) * 128], dtype=torch.float32),
"y_2": torch.tensor(y[i * 128:(i + 1) * 128]**2,
dtype=torch.float32),
"y_3": torch.tensor(y[i * 128:(i + 1) * 128]**2,
dtype=torch.float32)
}
} for i in range(1024 // 128)]
model = MultipleOutputModel()
transformations = {"y_1": Log10(), "y_2": Log10()}
losses = {
"y_1": Quantiles(np.linspace(0.05, 0.95, 10)),
"y_2": Mean(),
"y_3": Density(np.linspace(-2, 2, 11))
}
mrnn = MRNN(losses=losses, model=model)
metrics = [
"Bias", "CRPS", "MeanSquaredError", "ScatterPlot", "CalibrationPlot"
]
mrnn.train(batched_data,
validation_data=batched_data,
n_epochs=5,
keys=("x", "y"),
metrics=metrics)
def test_training_multi_mrnn():
"""
Ensure that training with batch objects as dicts and provided keys
argument works.
"""
set_default_backend("pytorch")
class MultipleOutputModel(nn.Module):
def __init__(self):
super().__init__()
self.hidden = nn.Linear(16, 128)
self.head_1 = nn.Linear(128, 10)
self.head_2 = nn.Linear(128, 1)
self.head_3 = nn.Linear(128, 20)
def forward(self, x):
x = torch.relu(self.hidden(x))
y_1 = self.head_1(x)
y_2 = self.head_2(x)
y_3 = self.head_3(x)
return {"y_1": y_1, "y_2": y_2, "y_3": y_3}
x = np.random.rand(2024, 16) + 1.0
y = np.sum(x, axis=-1)
y += np.random.normal(size=y.size)
batched_data = [{
"x":
torch.tensor(x[i * 128:(i + 1) * 128]).to(torch.float32),
"y": {
"y_1": torch.tensor(y[i * 128:(i + 1) * 128], dtype=torch.float32),
"y_2": torch.tensor(y[i * 128:(i + 1) * 128]**2,
dtype=torch.float32),
"y_3": torch.tensor(y[i * 128:(i + 1) * 128]**2,
dtype=torch.float32)
}
} for i in range(1024 // 128)]
model = MultipleOutputModel()
bins = np.linspace(0, 1, 12)
bins = {"y_1": bins, "y_2": bins}
losses = {
"y_1": Quantiles(np.linspace(0.05, 0.95, 10)),
"y_2": Mean(),
"y_3": Density(np.linspace(-2, 2, 21))
}
mrnn = MRNN(losses=losses, model=model)
mrnn.train(batched_data, n_epochs=1)
def test_training_with_dataloader():
"""
Ensure that training with a pytorch dataloader works.
"""
set_default_backend("keras")
x = np.random.rand(1024, 16)
y = np.random.rand(1024)
batched_data = [{
"x": x[i * 128:(i + 1) * 128],
"y": y[i * 128:(i + 1) * 128],
} for i in range(1024 // 128)]
qrnn = QRNN(np.linspace(0.05, 0.95, 10), n_inputs=x.shape[1])
qrnn.train(batched_data, n_epochs=1)
def test_qrnn_training_with_dataloader():
"""
Ensure that training with a pytorch dataloader works.
"""
set_default_backend("pytorch")
x = np.random.rand(1024, 16)
y = np.random.rand(1024)
training_data = torch.utils.data.TensorDataset(torch.tensor(x),
torch.tensor(y))
training_loader = torch.utils.data.DataLoader(training_data,
batch_size=128)
qrnn = QRNN(np.linspace(0.05, 0.95, 10), n_inputs=x.shape[1])
qrnn.train(training_loader, n_epochs=1)
def test_drnn(self, backend):
"""
Test training of DRNN using numpy arrays as input.
"""
set_default_backend(backend)
drnn = DRNN(self.bins, n_inputs=self.x_train.shape[1])
drnn.train((self.x_train, self.y_train),
validation_data=(self.x_train, self.y_train),
n_epochs=2)
drnn.predict(self.x_train)
mu = drnn.posterior_mean(self.x_train[:2, :])
assert len(mu.shape) == 1
r = drnn.sample_posterior(self.x_train[:4, :], n_samples=2)
assert r.shape == (4, 2)
def test_training_with_dict_and_keys():
"""
Ensure that training with batch objects as dicts and provided keys
argument works.
"""
set_default_backend("keras")
x = np.random.rand(1024, 16)
y = np.random.rand(1024)
batched_data = [{
"x": x[i * 128:(i + 1) * 128],
"x_2": x[i * 128:(i + 1) * 128],
"y": y[i * 128:(i + 1) * 128],
} for i in range(1024 // 128)]
qrnn = QRNN(np.linspace(0.05, 0.95, 10), n_inputs=x.shape[1])
qrnn.train(batched_data, n_epochs=1, keys=("x", "y"))
def test_qrnn_training_with_dict():
"""
Ensure that training with batch objects as dicts works.
"""
set_default_backend("pytorch")
x = np.random.rand(1024, 16)
y = np.random.rand(1024)
batched_data = [{
"x": torch.tensor(x[i * 128:(i + 1) * 128]),
"y": torch.tensor(y[i * 128:(i + 1) * 128]),
} for i in range(1024 // 128)]
qrnn = QRNN(np.linspace(0.05, 0.95, 10), n_inputs=x.shape[1])
qrnn.train(batched_data, n_epochs=1)
def test_save_drnn(self, backend):
"""
Test saving and loading of DRNNs.
"""
set_default_backend(backend)
drnn = DRNN(self.bins, n_inputs=self.x_train.shape[1])
f = tempfile.NamedTemporaryFile()
drnn.save(f.name)
drnn_loaded = DRNN.load(f.name)
x_pred = drnn.predict(self.x_train)
x_pred_loaded = drnn.predict(self.x_train)
if not type(x_pred) == np.ndarray:
x_pred = x_pred.detach()
assert np.allclose(x_pred, x_pred_loaded)
def test_save_qrnn(self, backend):
"""
Test saving and loading of QRNNs.
"""
set_default_backend(backend)
qrnn = QRNN(np.linspace(0.05, 0.95, 10),
n_inputs=self.x_train.shape[1])
f = tempfile.NamedTemporaryFile()
qrnn.save(f.name)
qrnn_loaded = QRNN.load(f.name)
x_pred = qrnn.predict(self.x_train)
x_pred_loaded = qrnn.predict(self.x_train)
if not type(x_pred) == np.ndarray:
x_pred = x_pred.detach()
assert np.allclose(x_pred, x_pred_loaded)
def test_qrnn_training_metrics():
"""
Ensure that training with a single target and metrics works.
"""
set_default_backend("pytorch")
x = np.random.rand(1024, 16)
y = np.random.rand(1024)
batched_data = [{
"x": torch.tensor(x[i * 128:(i + 1) * 128]),
"x_2": torch.tensor(x[i * 128:(i + 1) * 128]),
"y": torch.tensor(y[i * 128:(i + 1) * 128]),
} for i in range(1024 // 128)]
qrnn = QRNN(np.linspace(0.05, 0.95, 10), n_inputs=x.shape[1])
metrics = ["Bias", "MeanSquaredError", "CRPS"]
qrnn.train(batched_data, n_epochs=1, keys=("x", "y"), metrics=metrics)
def test_qrnn_training_metrics_conv():
"""
E
"""
set_default_backend("pytorch")
x_train = np.random.rand(
1024,
16,
32,
32,
)
y_train = np.random.rand(1024, 1, 32, 32)
x_val = np.random.rand(
32,
16,
32,
32,
)
y_val = np.random.rand(32, 1, 32, 32)
training_data = torch.utils.data.TensorDataset(torch.tensor(x_train),
torch.tensor(y_train))
training_loader = torch.utils.data.DataLoader(training_data,
batch_size=128)
validation_data = torch.utils.data.TensorDataset(torch.tensor(x_val),
torch.tensor(y_val))
validation_loader = torch.utils.data.DataLoader(validation_data,
batch_size=1)
model = nn.Sequential(nn.Conv2d(16, 10, 1))
qrnn = QRNN(np.linspace(0.05, 0.95, 10), model=model)
metrics = [
"Bias", "MeanSquaredError", "CRPS", "CalibrationPlot", "ScatterPlot"
]
qrnn.train(training_loader,
validation_data=validation_loader,
n_epochs=2,
metrics=metrics,
batch_size=1,
mask=-1)
def test_training_metrics_multi():
"""
Ensure that training with batch objects as dicts and provided keys
argument works.
"""
set_default_backend("pytorch")
class MultipleOutputModel(nn.Module):
def __init__(self):
super().__init__()
self.hidden = nn.Linear(16, 128)
self.head_1 = nn.Linear(128, 11)
self.head_2 = nn.Linear(128, 11)
def forward(self, x):
x = torch.relu(self.hidden(x))
y_1 = self.head_1(x)
y_2 = self.head_2(x)
return {"y_1": y_1, "y_2": y_2}
x = np.random.rand(2024, 16) + 1.0
y = np.sum(x, axis=-1)
y += np.random.normal(size=y.size)
batched_data = [{
"x": torch.tensor(x[i * 128:(i + 1) * 128]),
"y": {
"y_1": torch.tensor(y[i * 128:(i + 1) * 128]),
"y_2": torch.tensor(y[i * 128:(i + 1) * 128]**2)
}
} for i in range(1024 // 128)]
model = MultipleOutputModel()
bins = np.linspace(0, 1, 12)
bins = {"y_1": bins, "y_2": bins}
qrnn = DRNN(bins=bins, model=model)
metrics = [
"Bias", "MeanSquaredError", "CRPS", "ScatterPlot", "QuantileFunction"
]
qrnn.train(batched_data,
validation_data=batched_data,
n_epochs=5,
keys=("x", "y"),
metrics=metrics)
def test_drnn_dict_iterable(self, backend):
"""
Test training with dataset object that yields dicts instead of
tuples.
"""
set_default_backend(backend)
backend = get_default_backend()
class DictWrapper:
def __init__(self, data):
self.data = data
def __iter__(self):
for x, y in self.data:
yield {"x": x, "y": y}
def __len__(self):
return len(self.data)
data = backend.BatchedDataset((self.x_train, self.y_train), 256)
drnn = DRNN(self.bins, n_inputs=self.x_train.shape[1])
drnn.train(DictWrapper(data), n_epochs=2, keys=("x", "y"))
def test_mse_loss():
"""
Test masking for cross entropy loss.
"""
set_default_backend("pytorch")
y_pred = torch.rand(10, 10, 10)
y = torch.ones(10, 10, 10)
y[:, 0, :] = 0.55
loss = MSELoss(mask=-1.0)
ref = ((y_pred - y)**2).mean()
assert np.all(
np.isclose(
loss(y_pred, y).detach().numpy(),
ref.mean().detach().numpy()))
y[5:, :, :] = -1.0
y[:, :, 5:] = -1.0
ref = ((y_pred[:5, :, :5] - y[:5, :, :5])**2).mean()
assert np.all(
np.isclose(
loss(y_pred, y).detach().numpy(),
ref.mean().detach().numpy()))
