def make_simple_gluon_model(self):
net = self.make_net()
m = Model()
m.x = Variable(shape=(1, 1))
m.f = MXFusionGluonFunction(net, num_outputs=1)
m.y = m.f(m.x)
return m
def test_mean_argument(self):
with pytest.raises(ModelSpecificationError):
dtype = 'float64'
net = nn.HybridSequential(prefix='nn_')
with net.name_scope():
net.add(
nn.Dense(1,
flatten=False,
activation="tanh",
in_units=2,
dtype=dtype))
net.initialize(mx.init.Xavier(magnitude=3))
rbf = RBF(2, True, 1., 1., 'rbf', None, dtype)
X_var = Variable(shape=(5, 2))
X_cond_var = Variable(shape=(8, 2))
Y_cond_var = Variable(shape=(8, 1))
mean_func = MXFusionGluonFunction(net,
num_outputs=1,
broadcastable=True)
mean_var = mean_func(X_var)
mean_cond_var = mean_func(X_cond_var)
gp = ConditionalGaussianProcess.define_variable(
X=X_var,
X_cond=X_cond_var,
Y_cond=Y_cond_var,
mean_cond=mean_cond_var,
kernel=rbf,
shape=(5, 1),
dtype=dtype)
def test_add_unresolved_components_function(self):
f = MXFusionGluonFunction(self.basic_net, num_outputs=1)
x = mfc.Variable()
y = f(x)
component_set = set((x, y))
self.fg.y = y
self.assertTrue(component_set <= set(self.fg.components_graph.nodes().keys()),
"Variables are all added to _components_graph {} {}".format(component_set, self.fg.components_graph.nodes().keys()))
self.assertTrue(set(map(lambda x: x.uuid, component_set)) <= self.fg.components.keys(), "Variable is added to _components dict. {} {}".format(set(map(lambda x: x.uuid, component_set)), self.fg.components.keys()))
self.assertTrue(len(self.fg.components_graph.nodes().keys()) == 5, "There should be variables for the block parameters and a function evaluation totally 5 nodes in the graph but there were only {} in {}".format(len(self.fg.components_graph.nodes().keys()), self.fg.components_graph.nodes().keys()))
def make_model(self, net):
m = mf.models.Model(verbose=True)
m.N = mf.components.Variable()
m.f = MXFusionGluonFunction(net, nOutputs=1)
m.x = mf.components.Variable(shape=(m.N,1))
m.v = mf.components.Variable(shape=(1,), transformation=PositiveTransformation(), initial_value=mx.nd.array([0.01]))
m.prior_variance = mf.components.Variable(shape=(1,), transformation=PositiveTransformation())
m.r = m.f(m.x)
for _, v in m.r.factor.block_variables:
v.set_prior(mf.components.distributions.Normal(mean=mx.nd.array([0]),variance=m.prior_variance))
m.y = mf.components.distributions.Normal.define_variable(mean=m.r, variance=m.v, shape=(m.N,1))
return m
def make_model(self, net):
dtype = get_default_dtype()
m = mf.models.Model(verbose=False)
m.N = mf.components.Variable()
m.f = MXFusionGluonFunction(net, num_outputs=1)
m.x = mf.components.Variable(shape=(m.N, 1))
m.r = m.f(m.x)
for k, v in m.r.factor.parameters.items():
if k.endswith('_weight') or k.endswith('_bias'):
v.set_prior(mf.components.distributions.Normal(mean=mx.nd.array([0], dtype=dtype), variance=mx.nd.array([1e6], dtype=dtype)))
m.y = mf.components.distributions.Categorical.define_variable(log_prob=m.r, num_classes=2, normalization=True, one_hot_encoding=False, shape=(m.N, 1), dtype=dtype)
return m
def test_draw_samples_w_mean(self):
D, X, Y, noise_var, lengthscale, variance = self.gen_data()
dtype = 'float64'
rand_gen = MockMXNetRandomGenerator(
mx.nd.array(np.random.rand(20 * D), dtype=dtype))
m, net = self.gen_mxfusion_model_w_mean(dtype, D, noise_var,
lengthscale, variance,
rand_gen)
observed = [m.X]
infr = Inference(ForwardSamplingAlgorithm(m,
observed,
num_samples=2,
target_variables=[m.Y]),
dtype=dtype)
samples = infr.run(X=mx.nd.array(X, dtype=dtype),
Y=mx.nd.array(Y, dtype=dtype))[0].asnumpy()
kern = RBF(3, True, name='rbf', dtype=dtype) + White(3, dtype=dtype)
X_var = Variable(shape=(10, 3))
mean_func = MXFusionGluonFunction(net,
num_outputs=1,
broadcastable=True)
mean_var = mean_func(X_var)
gp = GaussianProcess.define_variable(X=X_var,
kernel=kern,
mean=mean_var,
shape=(10, D),
dtype=dtype,
rand_gen=rand_gen).factor
variables = {
gp.X.uuid:
mx.nd.expand_dims(mx.nd.array(X, dtype=dtype), axis=0),
gp.add_rbf_lengthscale.uuid:
mx.nd.expand_dims(mx.nd.array(lengthscale, dtype=dtype), axis=0),
gp.add_rbf_variance.uuid:
mx.nd.expand_dims(mx.nd.array(variance, dtype=dtype), axis=0),
gp.add_white_variance.uuid:
mx.nd.expand_dims(mx.nd.array(noise_var, dtype=dtype), axis=0),
mean_var.uuid:
mx.nd.expand_dims(net(mx.nd.array(X, dtype=dtype)), axis=0)
}
samples_2 = gp.draw_samples(F=mx.nd,
variables=variables,
num_samples=2).asnumpy()
assert np.allclose(samples, samples_2), (samples, samples_2)
def make_bnn_model(self, net):
component_set = set()
m = mf.models.Model(verbose=False)
m.N = mfc.Variable()
m.f = MXFusionGluonFunction(net, num_outputs=1)
m.x = mfc.Variable(shape=(m.N,))
m.r = m.f(m.x)
for k, v in m.r.factor.parameters.items():
if k.endswith('_weight') or k.endswith('_bias'):
d = mf.components.distributions.Normal(mean=mx.nd.array([0]), variance=mx.nd.array([1e6]))
v.set_prior(d)
component_set.union(set([v,d]))
m.y = mf.components.distributions.Categorical.define_variable(log_prob=m.r, num_classes=2, normalization=True, one_hot_encoding=False)
component_set.union(set([m.N, m.f, m.x, m.r, m.y]))
return m, component_set
def make_bnn_model(self, net):
dtype = get_default_dtype()
m = mf.models.Model(verbose=True)
m.N = mf.components.Variable()
m.f = MXFusionGluonFunction(net, num_outputs=1)
m.x = mf.components.Variable(shape=(m.N,1))
m.v = mf.components.Variable(shape=(1,), transformation=PositiveTransformation(), initial_value=0.01)
m.prior_variance = mf.components.Variable(shape=(1,), transformation=PositiveTransformation())
m.r = m.f(m.x)
for _, v in m.r.factor.parameters.items():
mean = broadcast_to(Variable(mx.nd.array([0], dtype=dtype)),
v.shape)
var = broadcast_to(m.prior_variance, v.shape)
v.set_prior(mf.components.distributions.Normal(mean=mean, variance=var))
m.y = mf.components.distributions.Normal.define_variable(mean=m.r, variance=broadcast_to(m.v, (m.N, 1)), shape=(m.N, 1))
return m
def test_replicate_function_only_self(self):
self.D = 10
self.net = nn.HybridSequential()
with self.net.name_scope():
self.net.add(nn.Dense(self.D, activation="relu"))
m = mf.models.Model(verbose=False)
f = MXFusionGluonFunction(self.net, num_outputs=1)
m.x = mfc.Variable()
m.y = f(m.x)
def func(component):
return 'recursive', 'recursive'
y2 = m.y.replicate(replication_function=func)
self.assertTrue(y2.factor is not None)
def gen_mxfusion_model_w_mean(self, dtype, D, Z, noise_var, lengthscale,
variance, rand_gen=None):
net = nn.HybridSequential(prefix='nn_')
with net.name_scope():
net.add(nn.Dense(D, flatten=False, activation="tanh",
in_units=3, dtype=dtype))
net.initialize(mx.init.Xavier(magnitude=3))
m = Model()
m.N = Variable()
m.X = Variable(shape=(m.N, 3))
m.Z = Variable(shape=(3, 3), initial_value=mx.nd.array(Z, dtype=dtype))
m.noise_var = Variable(transformation=PositiveTransformation(), initial_value=mx.nd.array(noise_var, dtype=dtype))
kernel = RBF(input_dim=3, ARD=True, variance=mx.nd.array(variance, dtype=dtype), lengthscale=mx.nd.array(lengthscale, dtype=dtype), dtype=dtype)
m.mean_func = MXFusionGluonFunction(net, num_outputs=1,
broadcastable=True)
m.Y = SparseGPRegression.define_variable(X=m.X, kernel=kernel, noise_var=m.noise_var, mean=m.mean_func(m.X), inducing_inputs=m.Z, shape=(m.N, D), dtype=dtype)
m.Y.factor.sgp_log_pdf.jitter = 1e-8
return m, net
def test_draw_samples_w_mean(self, dtype, X, X_isSamples, rbf_lengthscale,
rbf_lengthscale_isSamples, rbf_variance,
rbf_variance_isSamples, rv_shape,
num_samples):
net = nn.HybridSequential(prefix='nn_')
with net.name_scope():
net.add(
nn.Dense(rv_shape[-1],
flatten=False,
activation="tanh",
in_units=X.shape[-1],
dtype=dtype))
net.initialize(mx.init.Xavier(magnitude=3))
X_mx = prepare_mxnet_array(X, X_isSamples, dtype)
rbf_lengthscale_mx = prepare_mxnet_array(rbf_lengthscale,
rbf_lengthscale_isSamples,
dtype)
rbf_variance_mx = prepare_mxnet_array(rbf_variance,
rbf_variance_isSamples, dtype)
mean_mx = net(X_mx)
mean_np = mean_mx.asnumpy()
rand = np.random.randn(num_samples, *rv_shape)
rand_gen = MockMXNetRandomGenerator(
mx.nd.array(rand.flatten(), dtype=dtype))
rbf = RBF(2, True, 1., 1., 'rbf', None, dtype)
X_var = Variable(shape=(5, 2))
mean_func = MXFusionGluonFunction(net,
num_outputs=1,
broadcastable=True)
mean_var = mean_func(X_var)
gp = GaussianProcess.define_variable(X=X_var,
kernel=rbf,
shape=rv_shape,
mean=mean_var,
dtype=dtype,
rand_gen=rand_gen).factor
variables = {
gp.X.uuid: X_mx,
gp.rbf_lengthscale.uuid: rbf_lengthscale_mx,
gp.rbf_variance.uuid: rbf_variance_mx,
gp.mean.uuid: mean_mx
}
samples_rt = gp.draw_samples(F=mx.nd,
variables=variables,
num_samples=num_samples).asnumpy()
samples_np = []
for i in range(num_samples):
X_i = X[i] if X_isSamples else X
lengthscale_i = rbf_lengthscale[
i] if rbf_lengthscale_isSamples else rbf_lengthscale
variance_i = rbf_variance[
i] if rbf_variance_isSamples else rbf_variance
rand_i = rand[i]
rbf_np = GPy.kern.RBF(input_dim=2, ARD=True)
rbf_np.lengthscale = lengthscale_i
rbf_np.variance = variance_i
K_np = rbf_np.K(X_i)
L_np = np.linalg.cholesky(K_np)
sample_np = L_np.dot(rand_i)
samples_np.append(sample_np)
samples_np = np.array(samples_np) + mean_np
assert np.issubdtype(samples_rt.dtype, dtype)
assert get_num_samples(mx.nd, samples_rt) == num_samples
assert np.allclose(samples_np, samples_rt)
def test_log_pdf_w_mean(self, dtype, X, X_isSamples, rbf_lengthscale,
rbf_lengthscale_isSamples, rbf_variance,
rbf_variance_isSamples, rv, rv_isSamples,
num_samples):
net = nn.HybridSequential(prefix='nn_')
with net.name_scope():
net.add(
nn.Dense(rv.shape[-1],
flatten=False,
activation="tanh",
in_units=X.shape[-1],
dtype=dtype))
net.initialize(mx.init.Xavier(magnitude=3))
X_mx = prepare_mxnet_array(X, X_isSamples, dtype)
rbf_lengthscale_mx = prepare_mxnet_array(rbf_lengthscale,
rbf_lengthscale_isSamples,
dtype)
rbf_variance_mx = prepare_mxnet_array(rbf_variance,
rbf_variance_isSamples, dtype)
rv_mx = prepare_mxnet_array(rv, rv_isSamples, dtype)
rv_shape = rv.shape[1:] if rv_isSamples else rv.shape
mean_mx = net(X_mx)
mean_np = mean_mx.asnumpy()
rbf = RBF(2, True, 1., 1., 'rbf', None, dtype)
X_var = Variable(shape=(5, 2))
mean_func = MXFusionGluonFunction(net,
num_outputs=1,
broadcastable=True)
mean_var = mean_func(X_var)
gp = GaussianProcess.define_variable(X=X_var,
kernel=rbf,
shape=rv_shape,
mean=mean_var,
dtype=dtype).factor
variables = {
gp.X.uuid: X_mx,
gp.rbf_lengthscale.uuid: rbf_lengthscale_mx,
gp.rbf_variance.uuid: rbf_variance_mx,
gp.random_variable.uuid: rv_mx,
gp.mean.uuid: mean_mx
}
log_pdf_rt = gp.log_pdf(F=mx.nd, variables=variables).asnumpy()
log_pdf_np = []
for i in range(num_samples):
X_i = X[i] if X_isSamples else X
lengthscale_i = rbf_lengthscale[
i] if rbf_lengthscale_isSamples else rbf_lengthscale
variance_i = rbf_variance[
i] if rbf_variance_isSamples else rbf_variance
rv_i = rv[i] if rv_isSamples else rv
rv_i = rv_i - mean_np[i] if X_isSamples else rv_i - mean_np[0]
rbf_np = GPy.kern.RBF(input_dim=2, ARD=True)
rbf_np.lengthscale = lengthscale_i
rbf_np.variance = variance_i
K_np = rbf_np.K(X_i)
log_pdf_np.append(
multivariate_normal.logpdf(rv_i[:, 0], mean=None, cov=K_np))
log_pdf_np = np.array(log_pdf_np)
isSamples_any = any([
X_isSamples, rbf_lengthscale_isSamples, rbf_variance_isSamples,
rv_isSamples
])
assert np.issubdtype(log_pdf_rt.dtype, dtype)
assert array_has_samples(mx.nd, log_pdf_rt) == isSamples_any
if isSamples_any:
assert get_num_samples(mx.nd, log_pdf_rt) == num_samples
assert np.allclose(log_pdf_np, log_pdf_rt)
def test_draw_samples_w_mean(self, dtype, X, X_isSamples, X_cond,
X_cond_isSamples, Y_cond, Y_cond_isSamples,
rbf_lengthscale, rbf_lengthscale_isSamples,
rbf_variance, rbf_variance_isSamples,
rv_shape, num_samples):
net = nn.HybridSequential(prefix='nn_')
with net.name_scope():
net.add(
nn.Dense(rv_shape[-1],
flatten=False,
activation="tanh",
in_units=X.shape[-1],
dtype=dtype))
net.initialize(mx.init.Xavier(magnitude=3))
from scipy.linalg.lapack import dtrtrs
X_mx = prepare_mxnet_array(X, X_isSamples, dtype)
X_cond_mx = prepare_mxnet_array(X_cond, X_cond_isSamples, dtype)
Y_cond_mx = prepare_mxnet_array(Y_cond, Y_cond_isSamples, dtype)
rbf_lengthscale_mx = prepare_mxnet_array(rbf_lengthscale,
rbf_lengthscale_isSamples,
dtype)
rbf_variance_mx = prepare_mxnet_array(rbf_variance,
rbf_variance_isSamples, dtype)
mean_mx = net(X_mx)
mean_np = mean_mx.asnumpy()
mean_cond_mx = net(X_cond_mx)
mean_cond_np = mean_cond_mx.asnumpy()
rand = np.random.randn(num_samples, *rv_shape)
rand_gen = MockMXNetRandomGenerator(
mx.nd.array(rand.flatten(), dtype=dtype))
rbf = RBF(2, True, 1., 1., 'rbf', None, dtype)
X_var = Variable(shape=(5, 2))
X_cond_var = Variable(shape=(8, 2))
Y_cond_var = Variable(shape=(8, 1))
mean_func = MXFusionGluonFunction(net,
num_outputs=1,
broadcastable=True)
mean_var = mean_func(X_var)
mean_cond_var = mean_func(X_cond_var)
gp = ConditionalGaussianProcess.define_variable(
X=X_var,
X_cond=X_cond_var,
Y_cond=Y_cond_var,
mean=mean_var,
mean_cond=mean_cond_var,
kernel=rbf,
shape=rv_shape,
dtype=dtype,
rand_gen=rand_gen).factor
variables = {
gp.X.uuid: X_mx,
gp.X_cond.uuid: X_cond_mx,
gp.Y_cond.uuid: Y_cond_mx,
gp.rbf_lengthscale.uuid: rbf_lengthscale_mx,
gp.rbf_variance.uuid: rbf_variance_mx,
gp.mean.uuid: mean_mx,
gp.mean_cond.uuid: mean_cond_mx
}
samples_rt = gp.draw_samples(F=mx.nd,
variables=variables,
num_samples=num_samples).asnumpy()
samples_np = []
for i in range(num_samples):
X_i = X[i] if X_isSamples else X
X_cond_i = X_cond[i] if X_cond_isSamples else X_cond
Y_cond_i = Y_cond[i] if Y_cond_isSamples else Y_cond
Y_cond_i = Y_cond_i - mean_cond_np[
i] if X_cond_isSamples else Y_cond_i - mean_cond_np[0]
lengthscale_i = rbf_lengthscale[
i] if rbf_lengthscale_isSamples else rbf_lengthscale
variance_i = rbf_variance[
i] if rbf_variance_isSamples else rbf_variance
rand_i = rand[i]
rbf_np = GPy.kern.RBF(input_dim=2, ARD=True)
rbf_np.lengthscale = lengthscale_i
rbf_np.variance = variance_i
K_np = rbf_np.K(X_i)
Kc_np = rbf_np.K(X_cond_i, X_i)
Kcc_np = rbf_np.K(X_cond_i)
L = np.linalg.cholesky(Kcc_np)
LInvY = dtrtrs(L, Y_cond_i, lower=1, trans=0)[0]
LinvKxt = dtrtrs(L, Kc_np, lower=1, trans=0)[0]
mu = LinvKxt.T.dot(LInvY)
cov = K_np - LinvKxt.T.dot(LinvKxt)
L_cov_np = np.linalg.cholesky(cov)
sample_np = mu + L_cov_np.dot(rand_i)
samples_np.append(sample_np)
samples_np = np.array(samples_np) + mean_np
assert np.issubdtype(samples_rt.dtype, dtype)
assert get_num_samples(mx.nd, samples_rt) == num_samples
assert np.allclose(samples_np, samples_rt)