def _get_marginal_distribution(self, index_points=None, is_missing=None):
# TODO(cgs): consider caching the result here, keyed on `index_points`.
index_points = self._get_index_points(index_points)
covariance = self._compute_covariance(index_points)
is_univariate_marginal = self._is_univariate_marginal(index_points)
loc = self._mean_fn(index_points)
if is_univariate_marginal:
# `loc` has a trailing 1 in the shape; squeeze it.
loc = tf.squeeze(loc, axis=-1)
if is_missing is not None:
loc = tf.where(is_missing, 0., loc)
if is_univariate_marginal:
covariance = tf.where(is_missing, 1., covariance)
else:
covariance = psd_kernels_util.mask_matrix(
covariance, is_missing) # pylint:disable=invalid-unary-operand-type
# If we're sure the number of index points is 1, we can just construct a
# scalar Normal. This has computational benefits and supports things like
# CDF that aren't otherwise straightforward to provide.
if is_univariate_marginal:
scale = tf.sqrt(covariance)
return normal.Normal(loc=loc,
scale=scale,
validate_args=self._validate_args,
allow_nan_stats=self._allow_nan_stats,
name='marginal_distribution')
else:
return self._marginal_fn(loc=loc,
covariance=covariance,
validate_args=self._validate_args,
allow_nan_stats=self._allow_nan_stats,
name='marginal_distribution')
def _divisor_matrix(self, fixed_inputs=None, fixed_inputs_mask=None):
fixed_inputs = tf.convert_to_tensor(
self._fixed_inputs if fixed_inputs is None else fixed_inputs)
if fixed_inputs_mask is None:
fixed_inputs_mask = self._fixed_inputs_mask
# NOTE: Replacing masked-out rows/columns of the divisor matrix with
# rows/columns from the identity matrix is equivalent to using a divisor
# matrix in which those rows and columns have been dropped.
return util.mask_matrix(_compute_divisor_matrix(
self._base_kernel,
diag_shift=self._diag_shift,
fixed_inputs=fixed_inputs),
mask=fixed_inputs_mask)
def _compute_observation_scale(kernel,
observation_index_points,
cholesky_fn,
observation_noise_variance=None,
observations_is_missing=None):
"""Compute matrix square root of the kernel on observation index points."""
if observations_is_missing is not None:
observations_is_missing = tf.convert_to_tensor(observations_is_missing)
# If observations are missing, there's nothing we can do to preserve the
# operator structure, so densify.
observation_covariance = kernel.matrix_over_all_tasks(
observation_index_points, observation_index_points).to_dense()
if observation_noise_variance is not None:
broadcast_shape = distribution_util.get_broadcast_shape(
observation_covariance,
observation_noise_variance[..., tf.newaxis, tf.newaxis])
observation_covariance = tf.broadcast_to(observation_covariance,
broadcast_shape)
observation_covariance = _add_diagonal_shift(
observation_covariance, observation_noise_variance)
vec_observations_is_missing = _vec(observations_is_missing)
observation_covariance = tf.linalg.LinearOperatorFullMatrix(
psd_kernels_util.mask_matrix(
observation_covariance,
is_missing=vec_observations_is_missing),
is_non_singular=True,
is_positive_definite=True)
observation_scale = cholesky_util.cholesky_from_fn(
observation_covariance, cholesky_fn)
else:
observation_scale = mtgp._compute_flattened_scale( # pylint:disable=protected-access
kernel=kernel,
index_points=observation_index_points,
cholesky_fn=cholesky_fn,
observation_noise_variance=observation_noise_variance)
return observation_scale
def precompute_regression_model(
kernel,
observation_index_points,
observations,
observations_is_missing=None,
index_points=None,
observation_noise_variance=None,
predictive_noise_variance=None,
mean_fn=None,
cholesky_fn=None,
validate_args=False,
allow_nan_stats=False,
name='PrecomputedMultiTaskGaussianProcessRegressionModel'):
"""Returns a MTGaussianProcessRegressionModel with precomputed quantities.
This differs from the constructor by precomputing quantities associated with
observations in a non-tape safe way. `index_points` is the only parameter
that is allowed to vary (i.e. is a `Variable` / changes after
initialization).
Specifically:
* We make `observation_index_points` and `observations` mandatory
parameters.
* We precompute `kernel(observation_index_points, observation_index_points)`
along with any other associated quantities relating to the `kernel`,
`observations` and `observation_index_points`.
A typical usecase would be optimizing kernel hyperparameters for a
`MultiTaskGaussianProcess`, and computing the posterior predictive with
respect to those optimized hyperparameters and observation / index-points
pairs.
WARNING: This method assumes `index_points` is the only varying parameter
(i.e. is a `Variable` / changes after initialization) and hence is not
tape-safe.
Args:
kernel: `PositiveSemidefiniteKernel`-like instance representing the
GP's covariance function.
observation_index_points: `float` `Tensor` representing finite collection,
or batch of collections, of points in the index set for which some data
has been observed. Shape has the form `[b1, ..., bB, e, f1, ..., fF]`
where `F` is the number of feature dimensions and must equal
`kernel.feature_ndims`, and `e` is the number (size) of index points in
each batch. `[b1, ..., bB, e]` must be broadcastable with the shape of
`observations`, and `[b1, ..., bB]` must be broadcastable with the
shapes of all other batched parameters (`kernel.batch_shape`,
`index_points`, etc). The default value is `None`, which corresponds to
the empty set of observations, and simply results in the prior
predictive model (a GP with noise of variance
`predictive_noise_variance`).
observations: `float` `Tensor` representing collection, or batch of
collections, of observations corresponding to
`observation_index_points`. Shape has the form `[b1, ..., bB, e, t]`
The batch shape `[b1, ..., bB]` must be
broadcastable with the shapes of all other batched parameters
(`kernel.batch_shape`, `index_points`, etc.). The default value is
`None`, which corresponds to the empty set of observations, and simply
results in the prior predictive model (a GP with noise of variance
`predictive_noise_variance`).
observations_is_missing: `bool` `Tensor` of shape `[..., e]`,
representing a batch of boolean masks. When `observations_is_missing`
is not `None`, the returned distribution is conditioned only on the
observations for which the corresponding elements of
`observations_is_missing` are `True`.
index_points: `float` `Tensor` representing finite collection, or batch of
collections, of points in the index set over which the GP is defined.
Shape has the form `[b1, ..., bB, e, f1, ..., fF]` where `F` is the
number of feature dimensions and must equal `kernel.feature_ndims` and
`e` is the number (size) of index points in each batch. Ultimately this
distribution corresponds to an `e`-dimensional multivariate normal. The
batch shape must be broadcastable with `kernel.batch_shape` and any
batch dims yielded by `mean_fn`.
observation_noise_variance: `float` `Tensor` representing the variance
of the noise in the Normal likelihood distribution of the model. May be
batched, in which case the batch shape must be broadcastable with the
shapes of all other batched parameters (`kernel.batch_shape`,
`index_points`, etc.).
Default value: `None`
predictive_noise_variance: `float` `Tensor` representing the variance in
the posterior predictive model. If `None`, we simply re-use
`observation_noise_variance` for the posterior predictive noise. If set
explicitly, however, we use this value. This allows us, for example, to
omit predictive noise variance (by setting this to zero) to obtain
noiseless posterior predictions of function values, conditioned on noisy
observations.
mean_fn: Python `callable` that acts on `index_points` to produce a
collection, or batch of collections, of mean values at `index_points`.
Takes a `Tensor` of shape `[b1, ..., bB, f1, ..., fF]` and returns a
`Tensor` whose shape is broadcastable with `[b1, ..., bB, t]`.
Default value: `None` implies the constant zero function.
cholesky_fn: Callable which takes a single (batch) matrix argument and
returns a Cholesky-like lower triangular factor. Default value: `None`,
in which case `make_cholesky_with_jitter_fn` is used with the `jitter`
parameter.
validate_args: Python `bool`, default `False`. When `True` distribution
parameters are checked for validity despite possibly degrading runtime
performance. When `False` invalid inputs may silently render incorrect
outputs.
Default value: `False`.
allow_nan_stats: Python `bool`, default `True`. When `True`,
statistics (e.g., mean, mode, variance) use the value `NaN` to
indicate the result is undefined. When `False`, an exception is raised
if one or more of the statistic's batch members are undefined.
Default value: `False`.
name: Python `str` name prefixed to Ops created by this class.
Default value: 'PrecomputedGaussianProcessRegressionModel'.
Returns
An instance of `MultiTaskGaussianProcessRegressionModel` with precomputed
quantities associated with observations.
"""
with tf.name_scope(name) as name:
dtype = dtype_util.common_dtype([
index_points,
observation_index_points,
observations,
observation_noise_variance,
predictive_noise_variance,
], tf.float32)
# Convert-to-tensor arguments that are expected to not be Variables / not
# going to change.
observation_index_points = tf.convert_to_tensor(
observation_index_points, dtype=dtype)
if observation_noise_variance is not None:
observation_noise_variance = tf.convert_to_tensor(
observation_noise_variance, dtype=dtype)
observations = tf.convert_to_tensor(observations, dtype=dtype)
if observations_is_missing is not None:
observations_is_missing = tf.convert_to_tensor(
observations_is_missing)
if cholesky_fn is None:
cholesky_fn = cholesky_util.make_cholesky_with_jitter_fn()
else:
if not callable(cholesky_fn):
raise ValueError('`cholesky_fn` must be a Python callable')
if mean_fn is None:
mean_fn = lambda x: tf.zeros([1], dtype=dtype)
else:
if not callable(mean_fn):
raise ValueError('`mean_fn` must be a Python callable')
if observations_is_missing is not None:
# If observations are missing, there's nothing we can do to preserve the
# operator structure, so densify.
observation_covariance = kernel.matrix_over_all_tasks(
observation_index_points,
observation_index_points).to_dense()
if observation_noise_variance is not None:
broadcast_shape = distribution_util.get_broadcast_shape(
observation_covariance,
observation_noise_variance[..., tf.newaxis,
tf.newaxis])
observation_covariance = tf.broadcast_to(
observation_covariance, broadcast_shape)
observation_covariance = _add_diagonal_shift(
observation_covariance, observation_noise_variance)
vec_observations_is_missing = _vec(observations_is_missing)
observation_covariance = tf.linalg.LinearOperatorFullMatrix(
psd_kernels_util.mask_matrix(
observation_covariance,
is_missing=vec_observations_is_missing),
is_non_singular=True,
is_positive_definite=True)
observation_scale = cholesky_util.cholesky_from_fn(
observation_covariance, cholesky_fn)
else:
observation_scale = mtgp._compute_flattened_scale( # pylint:disable=protected-access
kernel=kernel,
index_points=observation_index_points,
cholesky_fn=cholesky_fn,
observation_noise_variance=observation_noise_variance)
# Note that the conditional mean is
# k(x, o) @ (k(o, o) + sigma**2)^-1 obs. We can precompute the latter
# term since it won't change per iteration.
vec_diff = _vec(observations - mean_fn(observation_index_points))
if observations_is_missing is not None:
vec_diff = tf.where(vec_observations_is_missing,
tf.zeros([], dtype=vec_diff.dtype),
vec_diff)
solve_on_observations = observation_scale.solvevec(
observation_scale.solvevec(vec_diff), adjoint=True)
def flattened_conditional_mean_fn(x):
return _flattened_conditional_mean_fn_helper(
x,
kernel,
observations,
observation_index_points,
observations_is_missing,
observation_scale,
mean_fn,
solve_on_observations=solve_on_observations)
mtgprm = MultiTaskGaussianProcessRegressionModel(
kernel=kernel,
observation_index_points=observation_index_points,
observations=observations,
index_points=index_points,
observation_noise_variance=observation_noise_variance,
predictive_noise_variance=predictive_noise_variance,
cholesky_fn=cholesky_fn,
_flattened_conditional_mean_fn=flattened_conditional_mean_fn,
_observation_scale=observation_scale,
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
name=name)
return mtgprm
def __init__(self,
kernel,
observation_index_points,
observations,
observations_is_missing=None,
index_points=None,
mean_fn=None,
observation_noise_variance=None,
predictive_noise_variance=None,
cholesky_fn=None,
validate_args=False,
allow_nan_stats=False,
name='MultiTaskGaussianProcessRegressionModelWithCholesky'):
"""Construct a MultiTaskGaussianProcessRegressionModelWithCholesky instance.
WARNING: This method assumes `index_points` is the only varying parameter
(i.e. is a `Variable` / changes after initialization) and hence is not
tape-safe.
Args:
kernel: `MultiTaskKernel`-like instance representing the GP's covariance
function.
observation_index_points: `float` `Tensor` representing finite collection,
or batch of collections, of points in the index set for which some data
has been observed. Shape has the form `[b1, ..., bB, e, f1, ..., fF]`
where `F` is the number of feature dimensions and must equal
`kernel.feature_ndims`, and `e` is the number (size) of index points in
each batch. `[b1, ..., bB, e]` must be broadcastable with the shape of
`observations`, and `[b1, ..., bB]` must be broadcastable with the
shapes of all other batched parameters (`kernel.batch_shape`,
`index_points`, etc).
observations: `float` `Tensor` representing collection, or batch of
collections, of observations corresponding to
`observation_index_points`. Shape has the form `[b1, ..., bB, e, t]`,
which must be broadcastable with the batch and example shapes of
`observation_index_points`. The batch shape `[b1, ..., bB]` must be
broadcastable with the shapes of all other batched parameters
(`kernel.batch_shape`, `index_points`, etc.).
observations_is_missing: `bool` `Tensor` of shape `[..., e, t]`,
representing a batch of boolean masks. When
`observations_is_missing` is not `None`, this distribution is
conditioned only on the observations for which the
corresponding elements of `observations_is_missing` are `False`.
index_points: `float` `Tensor` representing finite collection, or batch of
collections, of points in the index set over which the GP is defined.
Shape has the form `[b1, ..., bB, e, f1, ..., fF]` where `F` is the
number of feature dimensions and must equal `kernel.feature_ndims` and
`e` is the number (size) of index points in each batch. Ultimately this
distribution corresponds to an `e`-dimensional multivariate normal. The
batch shape must be broadcastable with `kernel.batch_shape`.
mean_fn: Python `callable` that acts on `index_points` to produce a (batch
of) collection of mean values at `index_points`. Takes a `Tensor` of
shape `[b1, ..., bB, e, f1, ..., fF]` and returns a `Tensor` whose shape
is broadcastable with `[b1, ..., bB, e, t]`, where `t` is the number of
tasks.
observation_noise_variance: `float` `Tensor` representing the variance of
the noise in the Normal likelihood distribution of the model. May be
batched, in which case the batch shape must be broadcastable with the
shapes of all other batched parameters (`kernel.batch_shape`,
`index_points`, etc.).
Default value: `None`
predictive_noise_variance: `float` `Tensor` representing the variance in
the posterior predictive model. If `None`, we simply re-use
`observation_noise_variance` for the posterior predictive noise. If set
explicitly, however, we use this value. This allows us, for example, to
omit predictive noise variance (by setting this to zero) to obtain
noiseless posterior predictions of function values, conditioned on noisy
observations.
cholesky_fn: Callable which takes a single (batch) matrix argument and
returns a Cholesky-like lower triangular factor. Default value: `None`,
in which case `make_cholesky_with_jitter_fn(1e-6)` is used.
validate_args: Python `bool`, default `False`. When `True` distribution
parameters are checked for validity despite possibly degrading runtime
performance. When `False` invalid inputs may silently render incorrect
outputs.
Default value: `False`.
allow_nan_stats: Python `bool`, default `True`. When `True`, statistics
(e.g., mean, mode, variance) use the value `NaN` to indicate the result
is undefined. When `False`, an exception is raised if one or more of the
statistic's batch members are undefined.
Default value: `False`.
name: Python `str` name prefixed to Ops created by this class.
Default value: 'MultiTaskGaussianProcessRegressionModel'.
"""
parameters = dict(locals())
with tf.name_scope(name) as name:
if not isinstance(kernel, multitask_kernel.MultiTaskKernel):
raise ValueError('`kernel` must be a `MultiTaskKernel`.')
dtype = dtype_util.common_dtype([
index_points, observation_index_points, observations,
observation_noise_variance, predictive_noise_variance
], tf.float32)
index_points = tensor_util.convert_nonref_to_tensor(
index_points, dtype=dtype, name='index_points')
observation_index_points = tf.convert_to_tensor(
observation_index_points,
dtype=dtype,
name='observation_index_points')
observations = tf.convert_to_tensor(
observations, dtype=dtype, name='observations')
if observations_is_missing is not None:
observations_is_missing = tf.convert_to_tensor(
observations_is_missing, dtype=tf.bool)
if observation_noise_variance is not None:
observation_noise_variance = tf.convert_to_tensor(
observation_noise_variance,
dtype=dtype,
name='observation_noise_variance')
predictive_noise_variance = tensor_util.convert_nonref_to_tensor(
predictive_noise_variance,
dtype=dtype,
name='predictive_noise_variance')
if predictive_noise_variance is None:
predictive_noise_variance = observation_noise_variance
if cholesky_fn is None:
self._cholesky_fn = cholesky_util.make_cholesky_with_jitter_fn()
else:
if not callable(cholesky_fn):
raise ValueError('`cholesky_fn` must be a Python callable')
self._cholesky_fn = cholesky_fn
self._kernel = kernel
self._index_points = index_points
# Scalar or vector the size of the number of tasks.
if mean_fn is not None:
if not callable(mean_fn):
raise ValueError('`mean_fn` must be a Python callable')
self._mean_fn = mean_fn
self._observation_noise_variance = observation_noise_variance
self._predictive_noise_variance = predictive_noise_variance
self._index_ponts = index_points
self._observation_index_points = observation_index_points
self._observations = observations
self._observations_is_missing = observations_is_missing
observation_covariance = self.kernel.matrix_over_all_tasks(
observation_index_points, observation_index_points)
if observation_noise_variance is not None:
observation_covariance = observation_covariance.to_dense()
broadcast_shape = distribution_util.get_broadcast_shape(
observation_covariance, observation_noise_variance[..., tf.newaxis,
tf.newaxis])
observation_covariance = tf.broadcast_to(observation_covariance,
broadcast_shape)
observation_covariance = _add_diagonal_shift(observation_covariance,
observation_noise_variance)
observation_covariance = tf.linalg.LinearOperatorFullMatrix(
observation_covariance,
is_non_singular=True,
is_positive_definite=True)
if observations_is_missing is not None:
vec_observations_is_missing = _vec(observations_is_missing)
observation_covariance = tf.linalg.LinearOperatorFullMatrix(
psd_kernels_util.mask_matrix(
observation_covariance.to_dense(),
mask=~vec_observations_is_missing),
is_non_singular=True,
is_positive_definite=True)
self._observation_cholesky = cholesky_util.cholesky_from_fn(
observation_covariance, self._cholesky_fn)
# Note that the conditional mean is
# k(x, o) @ (k(o, o) + sigma**2)^-1 obs. We can precompute the latter
# term since it won't change per iteration.
if mean_fn:
vec_observations = _vec(observations -
mean_fn(observation_index_points))
else:
vec_observations = _vec(observations)
if observations_is_missing is not None:
vec_observations = tf.where(~vec_observations_is_missing,
vec_observations,
tf.zeros([], dtype=vec_observations.dtype))
self._solve_on_obs = self._observation_cholesky.solvevec(
self._observation_cholesky.solvevec(vec_observations), adjoint=True)
super(MultiTaskGaussianProcessRegressionModel, self).__init__(
dtype=dtype,
reparameterization_type=(reparameterization.FULLY_REPARAMETERIZED),
validate_args=validate_args,
allow_nan_stats=allow_nan_stats,
parameters=parameters,
name=name)
def with_precomputed_divisor(base_kernel,
fixed_inputs,
fixed_inputs_mask=None,
diag_shift=None,
cholesky_fn=None,
validate_args=False,
name='PrecomputedSchurComplement'):
"""Returns a `SchurComplement` with a precomputed divisor matrix.
This method is the same as creating a `SchurComplement` kernel, but assumes
that `fixed_inputs`, `diag_shift` and `base_kernel` are unchanging /
not parameterized by any mutable state. We explicitly read / concretize
these values when this method is called, since we can precompute some
factorizations in order to speed up subsequent invocations of the kernel.
WARNING: This method assumes passed in arguments are not parameterized
by mutable state (`fixed_inputs`, `diag_shift` and `base_kernel`), and hence
is not tape-safe.
Args:
base_kernel: A `PositiveSemidefiniteKernel` instance, the kernel used to
build the block matrices of which this kernel computes the Schur
complement.
fixed_inputs: A Tensor, representing a collection of inputs. The Schur
complement that this kernel computes comes from a block matrix, whose
bottom-right corner is derived from `base_kernel.matrix(fixed_inputs,
fixed_inputs)`, and whose top-right and bottom-left pieces are
constructed by computing the base_kernel at pairs of input locations
together with these `fixed_inputs`. `fixed_inputs` is allowed to be an
empty collection (either `None` or having a zero shape entry), in which
case the kernel falls back to the trivial application of `base_kernel`
to inputs. See class-level docstring for more details on the exact
computation this does; `fixed_inputs` correspond to the `Z` structure
discussed there. `fixed_inputs` is assumed to have shape `[b1, ..., bB,
N, f1, ..., fF]` where the `b`'s are batch shape entries, the `f`'s are
feature_shape entries, and `N` is the number of fixed inputs. Use of
this kernel entails a 1-time O(N^3) cost of computing the Cholesky
decomposition of the k(Z, Z) matrix. The batch shape elements of
`fixed_inputs` must be broadcast compatible with
`base_kernel.batch_shape`.
fixed_inputs_mask: A boolean Tensor of shape `[..., N]`. When `mask` is
not None and an element of `mask` is False, the returned kernel will
return values computed as if the divisor matrix did not contain the
corresponding row or column.
diag_shift: A floating point scalar to be added to the diagonal of the
divisor_matrix before computing its Cholesky.
cholesky_fn: Callable which takes a single (batch) matrix argument and
returns a Cholesky-like lower triangular factor. Default value: `None`,
in which case `make_cholesky_with_jitter_fn` is used with the `jitter`
parameter.
validate_args: If `True`, parameters are checked for validity despite
possibly degrading runtime performance.
Default value: `False`
name: Python `str` name prefixed to Ops created by this class.
Default value: `"PrecomputedSchurComplement"`
"""
dtype = dtype_util.common_dtype(
[base_kernel, fixed_inputs, diag_shift], tf.float32)
fixed_inputs = tf.convert_to_tensor(fixed_inputs, dtype)
if fixed_inputs_mask is not None:
fixed_inputs_mask = tf.convert_to_tensor(fixed_inputs_mask,
tf.bool)
if diag_shift is not None:
diag_shift = tf.convert_to_tensor(diag_shift, dtype)
if cholesky_fn is None:
from tensorflow_probability.python.distributions import cholesky_util # pylint:disable=g-import-not-at-top
cholesky_fn = cholesky_util.make_cholesky_with_jitter_fn()
# TODO(b/196219597): Add a check to ensure that we have a `base_kernel`
# that is explicitly concretized.
divisor_matrix_cholesky = cholesky_fn(
util.mask_matrix(_compute_divisor_matrix(
base_kernel, diag_shift=diag_shift, fixed_inputs=fixed_inputs),
mask=fixed_inputs_mask))
schur_complement = SchurComplement(
base_kernel=base_kernel,
fixed_inputs=fixed_inputs,
fixed_inputs_mask=fixed_inputs_mask,
diag_shift=diag_shift,
cholesky_fn=cholesky_fn,
validate_args=validate_args,
_precomputed_divisor_matrix_cholesky=divisor_matrix_cholesky,
name=name)
return schur_complement