def w():
# In this module, weights are always required.
return B.rand(10, 2) + 1e-2
def test_update_inputs():
graph = Graph()
f = GP(EQ(), graph=graph)
x = array([[1], [2], [3]])
y = array([[4], [5], [6]])
res = B.concat([x, y], axis=1)
x_ind = array([[6], [7]])
res_ind = array([[6, 0], [7, 0]])
# Check vanilla case.
gpar = GPAR(x_ind=x_ind)
yield allclose, gpar._update_inputs(x, x_ind, y, f, None), (res, res_ind)
# Check imputation with prior.
gpar = GPAR(impute=True, x_ind=x_ind)
this_y = y.clone()
this_y[1] = np.nan
this_res = res.clone()
this_res[1, 1] = 0
yield allclose, \
gpar._update_inputs(x, x_ind, this_y, f, None), \
(this_res, res_ind)
# Check replacing with prior.
gpar = GPAR(replace=True, x_ind=x_ind)
this_y = y.clone()
this_y[1] = np.nan
this_res = res.clone()
this_res[0, 1] = 0
this_res[1, 1] = np.nan
this_res[2, 1] = 0
yield allclose, \
gpar._update_inputs(x, x_ind, this_y, f, None), \
(this_res, res_ind)
# Check imputation and replacing with prior.
gpar = GPAR(impute=True, replace=True, x_ind=x_ind)
this_res = res.clone()
this_res[:, 1] = 0
yield allclose, \
gpar._update_inputs(x, x_ind, y, f, None), \
(this_res, res_ind)
# Construct observations and update result for inducing points.
obs = Obs(f(array([1, 2, 3, 6, 7])), array([9, 10, 11, 12, 13]))
res_ind = array([[6, 12], [7, 13]])
# Check imputation with posterior.
gpar = GPAR(impute=True, x_ind=x_ind)
this_y = y.clone()
this_y[1] = np.nan
this_res = res.clone()
this_res[1, 1] = 10
yield allclose, \
gpar._update_inputs(x, x_ind, this_y, f, obs), \
(this_res, res_ind)
# Check replacing with posterior.
gpar = GPAR(replace=True, x_ind=x_ind)
this_y = y.clone()
this_y[1] = np.nan
this_res = res.clone()
this_res[0, 1] = 9
this_res[1, 1] = np.nan
this_res[2, 1] = 11
yield allclose, \
gpar._update_inputs(x, x_ind, this_y, f, obs), \
(this_res, res_ind)
# Check imputation and replacing with posterior.
gpar = GPAR(impute=True, replace=True, x_ind=x_ind)
this_res = res.clone()
this_res[0, 1] = 9
this_res[1, 1] = 10
this_res[2, 1] = 11
yield allclose, \
gpar._update_inputs(x, x_ind, y, f, obs), \
(this_res, res_ind)
def test_update_inputs():
prior = Measure()
f = GP(EQ(), measure=prior)
x = np.array([[1], [2], [3]])
y = np.array([[4], [5], [6]], dtype=float)
res = B.concat(x, y, axis=1)
x_ind = np.array([[6], [7]])
res_ind = np.array([[6, 0], [7, 0]])
# Check vanilla case.
gpar = GPAR(x_ind=x_ind)
approx(gpar._update_inputs(x, x_ind, y, f, None), (res, res_ind))
# Check imputation with prior.
gpar = GPAR(impute=True, x_ind=x_ind)
this_y = y.copy()
this_y[1] = np.nan
this_res = res.copy()
this_res[1, 1] = 0
approx(gpar._update_inputs(x, x_ind, this_y, f, None), (this_res, res_ind))
# Check replacing with prior.
gpar = GPAR(replace=True, x_ind=x_ind)
this_y = y.copy()
this_y[1] = np.nan
this_res = res.copy()
this_res[0, 1] = 0
this_res[1, 1] = np.nan
this_res[2, 1] = 0
approx(gpar._update_inputs(x, x_ind, this_y, f, None), (this_res, res_ind))
# Check imputation and replacing with prior.
gpar = GPAR(impute=True, replace=True, x_ind=x_ind)
this_res = res.copy()
this_res[:, 1] = 0
approx(gpar._update_inputs(x, x_ind, y, f, None), (this_res, res_ind))
# Construct observations and update result for inducing points.
obs = Obs(f(np.array([1, 2, 3, 6, 7])), np.array([9, 10, 11, 12, 13]))
res_ind = np.array([[6, 12], [7, 13]])
# Check imputation with posterior.
gpar = GPAR(impute=True, x_ind=x_ind)
this_y = y.copy()
this_y[1] = np.nan
this_res = res.copy()
this_res[1, 1] = 10
approx(gpar._update_inputs(x, x_ind, this_y, f, obs), (this_res, res_ind))
# Check replacing with posterior.
gpar = GPAR(replace=True, x_ind=x_ind)
this_y = y.copy()
this_y[1] = np.nan
this_res = res.copy()
this_res[0, 1] = 9
this_res[1, 1] = np.nan
this_res[2, 1] = 11
approx(gpar._update_inputs(x, x_ind, this_y, f, obs), (this_res, res_ind))
# Check imputation and replacing with posterior.
gpar = GPAR(impute=True, replace=True, x_ind=x_ind)
this_res = res.copy()
this_res[0, 1] = 9
this_res[1, 1] = 10
this_res[2, 1] = 11
approx(gpar._update_inputs(x, x_ind, y, f, obs), (this_res, res_ind))
def x(request):
# In this module, weights are always rank-two tensors.
d = request.param
return B.randn(10, d)
def test_log_transform():
x = B.rand(5)
f, f_inv = log_transform
approx(f(f_inv(x)), x)
def test_squishing_transform():
x = B.randn(5)
f, f_inv = squishing_transform
approx(f(f_inv(x)), x)
def w(request):
use_w = request.param
if use_w:
return B.rand(10, 2) + 1
else:
return None
from varz import Vars
from varz.torch import minimise_l_bfgs_b
from wbml.out import Counter
from .model import GPAR, per_output
__all__ = ['GPARRegressor', 'log_transform', 'squishing_transform']
log = logging.getLogger(__name__)
_dispatch = Dispatcher()
#: Log transform for the data.
log_transform = (B.log, B.exp)
#: Squishing transform for the data.
squishing_transform = (lambda x: B.sign(x) * B.log(1 + B.abs(x)),
lambda x: B.sign(x) * (B.exp(B.abs(x)) - 1))
def _vector_from_init(init, length):
# If only a single value is given, create ones.
if np.size(init) == 1:
return init * np.ones(length)
# Multiple values are given. Check that enough values are available.
init_squeezed = np.squeeze(init)
if np.ndim(init_squeezed) != 1:
raise ValueError('Incorrect shape {} of hyperparameters.'
''.format(np.shape(init)))
if np.size(init_squeezed) < length: # Squeezed doesn't change size.
raise ValueError('Not enough hyperparameters specified.')
def sample(self,
x,
w=None,
p=None,
posterior=False,
num_samples=1,
latent=False):
"""Sample from the prior or posterior.
Args:
x (matrix): Inputs to sample at.
w (matrix, optional): Weights of inputs to sample at.
p (int, optional): Number of outputs to sample if sampling from
the prior.
posterior (bool, optional): Sample from the prior instead of the
posterior.
num_samples (int, optional): Number of samples. Defaults to `1`.
latent (int, optional): Sample the latent function instead of
observations. Defaults to `False`.
Returns:
list[tensor]: Prior samples. If only a single sample is
generated, it will be returned directly instead of in a list.
"""
x = B.uprank(_to_torch(x))
# Check that model is conditioned or fit if sampling from the posterior.
if posterior and not self.is_conditioned:
raise RuntimeError(
"Must condition or fit model before sampling from the posterior."
)
# Check that the number of outputs is specified if sampling from the
# prior.
elif not posterior and p is None:
raise ValueError("Must specify number of outputs to sample.")
# Initialise weights.
if w is None:
w = B.ones(torch.float64,
B.shape(x)[0], self.p if posterior else p)
else:
w = B.uprank(_to_torch(w))
if posterior:
# Construct posterior GPAR.
gpar = _construct_gpar(self, self.vs, self.m, self.p)
gpar = gpar | (self.x, self.y, self.w)
else:
# Construct prior GPAR.
gpar = _construct_gpar(self, self.vs, B.shape(x)[1], p)
# Construct function to undo normalisation and transformation.
def undo_transforms(y_):
return self._untransform_y(self._unnormalise_y(y_))
# Perform sampling.
samples = []
with Counter(name="Sampling", total=num_samples) as counter:
for _ in range(num_samples):
counter.count()
samples.append(
undo_transforms(gpar.sample(
x, w, latent=latent)).detach_().numpy())
return samples[0] if num_samples == 1 else samples
def x(request):
shape = request.param
return B.randn(*shape)
def unnormalise_y(y_):
return B.add(B.multiply(y_, stds), means)
def normalise_y(y_):
return B.divide(B.subtract(y_, means), stds)
def __init__(
self,
replace=False,
impute=True,
scale=1.0,
scale_tie=False,
per=False,
per_period=1.0,
per_scale=1.0,
per_decay=10.0,
input_linear=False,
input_linear_scale=100.0,
linear=True,
linear_scale=100.0,
nonlinear=False,
nonlinear_scale=1.0,
rq=False,
markov=None,
noise=0.1,
x_ind=None,
normalise_y=True,
transform_y=(lambda x: x, lambda x: x),
):
# Model configuration.
self.replace = replace
self.impute = impute
self.sparse = x_ind is not None
if x_ind is None:
self.x_ind = None
else:
self.x_ind = B.uprank(_to_torch(x_ind))
self.model_config = {
"scale": scale,
"scale_tie": scale_tie,
"per": per,
"per_period": per_period,
"per_scale": per_scale,
"per_decay": per_decay,
"input_linear": input_linear,
"input_linear_scale": input_linear_scale,
"linear": linear,
"linear_scale": linear_scale,
"nonlinear": nonlinear,
"nonlinear_scale": nonlinear_scale,
"rq": rq,
"markov": markov,
"noise": noise,
}
# Model fitting.
self.vs = Vars(dtype=torch.float64)
self.is_conditioned = False
self.x = None # Inputs of training data
self.y = None # Outputs of training data
self.w = None # Weights for every time stamp
self.n = None # Number of data points
self.m = None # Number of input features
self.p = None # Number of outputs
# Output normalisation and transformation.
self.normalise_y = normalise_y
self._unnormalise_y, self._normalise_y = lambda x: x, lambda x: x
self._transform_y, self._untransform_y = transform_y
from wbml.out import Counter
from .model import GPAR, per_output
__all__ = ["GPARRegressor", "log_transform", "squishing_transform"]
log = logging.getLogger(__name__)
_dispatch = Dispatcher()
#: Log transform for the data.
log_transform = (B.log, B.exp)
#: Squishing transform for the data.
squishing_transform = (
lambda x: B.sign(x) * B.log(B.add(1, B.abs(x))),
lambda x: B.sign(x) * B.subtract(B.exp(B.abs(x)), 1),
)
def _vector_from_init(init, length):
# If only a single value is given, create ones.
if np.size(init) == 1:
return init * np.ones(length)
# Multiple values are given. Check that enough values are available.
init_squeezed = np.squeeze(init)
if np.ndim(init_squeezed) != 1:
raise ValueError("Incorrect shape {} of hyperparameters."
"".format(np.shape(init)))
if np.size(init_squeezed) < length: # Squeezed doesn't change size.
def test_inducing_points_uprank():
reg = GPARRegressor(x_ind=B.linspace(0, 10, 20))
assert reg.x_ind is not None
assert B.rank(reg.x_ind) == 2
def _init_weights(w, y):
if w is None:
return B.ones(torch.float64, *B.shape(y))
else:
return B.uprank(_to_torch(w))
def fit(self, x, y, greedy=False, fix=True, **kw_args):
"""Fit the model to data.
Further takes in keyword arguments for `Varz.minimise_l_bfgs_b`.
Args:
x (tensor): Inputs of training data.
y (tensor): Outputs of training data.
greedy (bool, optional): Greedily determine the ordering of the
outputs. Defaults to `False`.
fix (bool, optional): Fix the parameters of a layer after
training it. If set to `False`, the likelihood are
accumulated and all parameters are optimised at every step.
Defaults to `True`.
"""
if greedy:
raise NotImplementedError('Greedy search is not implemented yet.')
# Store data.
self.x = torch.tensor(B.uprank(x))
self.y = torch.tensor(self._transform_y(B.uprank(y)))
self.n, self.m = self.x.shape
self.p = self.y.shape[1]
# Perform normalisation, carefully handling missing values.
if self.normalise_y:
means, stds = [], []
for i in range(self.p):
# Filter missing observations.
available = ~B.isnan(self.y[:, i])
y_i = self.y[available, i]
# Calculate mean.
means.append(B.mean(y_i))
# Calculate std: safely handle the zero case.
std = B.std(y_i)
if std > 0:
stds.append(std)
else:
stds.append(B.cast(B.dtype(std), 1))
# Stack into a vector and create normalisers.
means, stds = B.stack(*means)[None, :], B.stack(*stds)[None, :]
def normalise_y(y_):
return (y_ - means) / stds
def unnormalise_y(y_):
return y_ * stds + means
# Save normalisers.
self._normalise_y = normalise_y
self._unnormalise_y = unnormalise_y
# Perform normalisation.
self.y = normalise_y(self.y)
# Precompute the results of `per_output`. This can otherwise incur a
# significant overhead if the number of outputs is large.
y_cached = {k: list(per_output(self.y, keep=k)) for k in [True, False]}
# Fit layer by layer.
# Note: `_construct_gpar` takes in the *number* of outputs.
sys.stdout.write('Training conditionals (total: {}):'.format(self.p))
sys.stdout.flush()
for pi in range(self.p):
sys.stdout.write(' {}'.format(pi + 1))
sys.stdout.flush()
# If we fix parameters of previous layers, we can precompute the
# inputs. This speeds up the optimisation massively.
if fix:
gpar = _construct_gpar(self, self.vs, self.m, pi + 1)
fixed_x, fixed_x_ind = gpar.logpdf(self.x,
y_cached,
only_last_layer=True,
outputs=list(range(pi)),
return_inputs=True)
def objective(vs):
gpar = _construct_gpar(self, vs, self.m, pi + 1)
# If the parameters of the previous layers are fixed, use the
# precomputed inputs.
if fix:
return -gpar.logpdf(fixed_x,
y_cached,
only_last_layer=True,
outputs=[pi],
x_ind=fixed_x_ind)
else:
return -gpar.logpdf(
self.x, y_cached, only_last_layer=False)
# Determine names to optimise.
if fix:
names = ['{}/*'.format(pi)]
else:
names = ['{}/*'.format(i) for i in range(pi + 1)]
# Perform the optimisation.
minimise_l_bfgs_b(objective, self.vs, names=names, **kw_args)
# Print newline to end progress bar.
sys.stdout.write('\n')
# Store that the model is fit.
self.is_fit = True