def test_softmax_y_shape_assert(num, dimF, dimY):
"""
SoftMax assumes the class is given as a label (not, e.g., one-hot
encoded), and hence just uses the first column of Y. To prevent
silent errors, there is a tf assertion that ensures Y only has one
dimension. This test checks that this assert works as intended.
"""
F = tf.random.normal((num, dimF))
dY = np.vstack((np.random.randn(num - 3, dimY), np.ones((3, dimY)))) > 0
Y = tf.convert_to_tensor(dY, dtype=default_int())
likelihood = Softmax(dimF)
try:
likelihood.log_prob(F, Y)
except tf.errors.InvalidArgumentError as e:
assert "Condition x == y did not hold." in e.message
def __init__(self,
gating_function_list: List[SVGPGatingFunction] = None,
likelihood: Likelihood = None,
name='GatingNetwork'):
super().__init__(gating_function_list, name=name)
# assert isinstance(gating_function_list, List)
# for gating_function in gating_function_list:
# assert isinstance(gating_function, SVGPGatingFunction)
# self.gating_function_list = gating_function_list
# self.num_experts = len(gating_function_list)
if likelihood is None:
self.likelihood = Softmax(num_classes=self.num_experts)
# self.likelihood = Softmax(num_classes=1)
else:
self.likelihood = likelihood
def test_softmax_bernoulli_equivalence(num, dimF, dimY):
dF = np.vstack(
(np.random.randn(num - 3,
dimF), np.array([[-3.0, 0.0], [3, 0.0], [0.0, 0.0]])))
dY = np.vstack((np.random.randn(num - 3, dimY), np.ones((3, dimY)))) > 0
F = to_default_float(dF)
Fvar = tf.exp(
tf.stack([F[:, 1], -10.0 + tf.zeros(F.shape[0], dtype=F.dtype)],
axis=1))
F = tf.stack([F[:, 0], tf.zeros(F.shape[0], dtype=F.dtype)], axis=1)
Y = to_default_int(dY)
Ylabel = 1 - Y
softmax_likelihood = Softmax(dimF)
bernoulli_likelihood = Bernoulli(invlink=tf.sigmoid)
softmax_likelihood.num_monte_carlo_points = int(
0.3e7) # Minimum number of points to pass the test on CircleCI
bernoulli_likelihood.num_gauss_hermite_points = 40
assert_allclose(
softmax_likelihood.conditional_mean(F)[:, :1],
bernoulli_likelihood.conditional_mean(F[:, :1]),
)
assert_allclose(
softmax_likelihood.conditional_variance(F)[:, :1],
bernoulli_likelihood.conditional_variance(F[:, :1]),
)
assert_allclose(
softmax_likelihood.log_prob(F, Ylabel),
bernoulli_likelihood.log_prob(F[:, :1], Y.numpy()),
)
mean1, var1 = softmax_likelihood.predict_mean_and_var(F, Fvar)
mean2, var2 = bernoulli_likelihood.predict_mean_and_var(
F[:, :1], Fvar[:, :1])
assert_allclose(mean1[:, 0, None], mean2, rtol=2e-3)
assert_allclose(var1[:, 0, None], var2, rtol=2e-3)
ls_ve = softmax_likelihood.variational_expectations(F, Fvar, Ylabel)
lb_ve = bernoulli_likelihood.variational_expectations(
F[:, :1], Fvar[:, :1], Y.numpy())
assert_allclose(ls_ve, lb_ve, rtol=5e-3)
class SVGPGatingNetworkMulti(SVGPGatingNetworkBase):
# TODO either remove likelihood or use Bernoulli/Softmax
def __init__(self,
gating_function_list: List[SVGPGatingFunction] = None,
likelihood: Likelihood = None,
name='GatingNetwork'):
super().__init__(gating_function_list, name=name)
# assert isinstance(gating_function_list, List)
# for gating_function in gating_function_list:
# assert isinstance(gating_function, SVGPGatingFunction)
# self.gating_function_list = gating_function_list
# self.num_experts = len(gating_function_list)
if likelihood is None:
self.likelihood = Softmax(num_classes=self.num_experts)
# self.likelihood = Softmax(num_classes=1)
else:
self.likelihood = likelihood
def predict_mixing_probs(self,
Xnew: InputData,
num_inducing_samples: int = None) -> tf.Tensor:
mixing_probs = []
Fmu, Fvar = [], []
for gating_function in self.gating_function_list:
# num_inducing_samples = None
f_mu, f_var = gating_function.predict_f(Xnew, num_inducing_samples)
Fmu.append(f_mu)
Fvar.append(f_var)
# Fmu = tf.stack(Fmu)
# Fvar = tf.stack(Fvar)
Fmu = tf.stack(Fmu, -1)
Fvar = tf.stack(Fvar, -1)
# Fmu = tf.concat(Fmu, -1)
# Fvar = tf.concat(Fvar, -1)
if num_inducing_samples is None:
Fmu = tf.transpose(Fmu, [1, 0, 2])
Fvar = tf.transpose(Fvar, [1, 0, 2])
else:
Fmu = tf.transpose(Fmu, [2, 0, 1, 3])
Fvar = tf.transpose(Fvar, [2, 0, 1, 3])
# TODO output dimension is always 1 so delete this
def single_output_predict_mean(args):
Fmu, Fvar = args
def single_predict_mean(args):
Fmu, Fvar = args
integrand2 = lambda *X: self.likelihood.conditional_variance(
*X) + tf.square(self.likelihood.conditional_mean(*X))
epsilon = None
E_y, E_y2 = ndiag_mc(
[self.likelihood.conditional_mean, integrand2],
S=self.likelihood.num_monte_carlo_points,
Fmu=Fmu,
Fvar=Fvar,
epsilon=epsilon)
return E_y
if num_inducing_samples is None:
mixing_probs = self.likelihood.predict_mean_and_var(Fmu,
Fvar)[0]
else:
# mixing_probs = tf.map_fn(single_predict_mean, (Fmu, Fvar),
# dtype=tf.float64)
mixing_probs = tf.vectorized_map(single_predict_mean,
(Fmu, Fvar))
return mixing_probs
# mixing_probs = tf.map_fn(single_output_predict_mean, (Fmu, Fvar),
# dtype=tf.float64)
mixing_probs = tf.vectorized_map(single_output_predict_mean,
(Fmu, Fvar))
if num_inducing_samples is None:
mixing_probs = tf.transpose(mixing_probs, [1, 0, 2])
else:
mixing_probs = tf.transpose(mixing_probs, [1, 2, 0, 3])
# mixing_probs = tf.transpose(mixing_probs, [1, 2, 0, 3])
return mixing_probs