def get_ppca_grad(self, x_train, inf_type, num_samples=100):
import random
dtype = get_default_dtype()
random.seed(0)
np.random.seed(0)
mx.random.seed(0)
m = self.make_ppca_model()
q = self.make_ppca_post(m)
observed = [m.x]
alg = inf_type(num_samples=num_samples,
model=m,
posterior=q,
observed=observed)
from mxfusion.inference.grad_based_inference import GradBasedInference
from mxfusion.inference import BatchInferenceLoop
infr = GradBasedInference(inference_algorithm=alg,
grad_loop=BatchInferenceLoop())
infr.initialize(x=mx.nd.array(x_train, dtype=dtype))
infr.run(max_iter=1,
learning_rate=1e-2,
x=mx.nd.array(x_train, dtype=dtype),
verbose=False)
return infr, q.post_mean
def test_score_function_rb_minibatch(self):
dtype = get_default_dtype()
x = np.random.rand(1000, 1)
y = np.random.rand(1000, 1)
x_nd, y_nd = mx.nd.array(y, dtype=dtype), mx.nd.array(x, dtype=dtype)
self.net = self.make_net()
self.net(x_nd)
m = self.make_bnn_model(self.net)
from mxfusion.inference.meanfield import create_Gaussian_meanfield
from mxfusion.inference.grad_based_inference import GradBasedInference
from mxfusion.inference import MinibatchInferenceLoop
observed = [m.y, m.x]
q = create_Gaussian_meanfield(model=m, observed=observed)
alg = ScoreFunctionRBInference(num_samples=3,
model=m,
observed=observed,
posterior=q)
infr = GradBasedInference(inference_algorithm=alg,
grad_loop=MinibatchInferenceLoop(
batch_size=100, rv_scaling={m.y: 10}))
infr.initialize(y=(100, 1), x=(100, 1))
infr.run(max_iter=1, learning_rate=1e-2, y=y_nd, x=x_nd)
def test_meanfield_saving(self):
dtype = get_default_dtype()
x = np.random.rand(10, 1)
y = np.random.rand(10, 1)
x_nd, y_nd = mx.nd.array(y, dtype=dtype), mx.nd.array(x, dtype=dtype)
self.net = self.make_net()
self.net(x_nd)
m = self.make_model(self.net)
from mxfusion.inference.meanfield import create_Gaussian_meanfield
from mxfusion.inference import StochasticVariationalInference
from mxfusion.inference.grad_based_inference import GradBasedInference
from mxfusion.inference import BatchInferenceLoop
observed = [m.y, m.x]
q = create_Gaussian_meanfield(model=m, observed=observed)
alg = StochasticVariationalInference(num_samples=3,
model=m,
observed=observed,
posterior=q)
infr = GradBasedInference(inference_algorithm=alg,
grad_loop=BatchInferenceLoop())
infr.initialize(y=y_nd, x=x_nd)
infr.run(max_iter=1, learning_rate=1e-2, y=y_nd, x=x_nd)
infr.save(self.ZIPNAME)
os.remove(self.ZIPNAME)
def make_ppca_post(self, m):
from mxfusion.inference import BatchInferenceLoop, GradBasedInference
dtype = get_default_dtype()
class SymmetricMatrix(mx.gluon.HybridBlock):
def hybrid_forward(self, F, x, *args, **kwargs):
return F.sum((F.expand_dims(x, 3) * F.expand_dims(x, 2)),
axis=-3)
q = mf.models.Posterior(m)
sym = mf.components.functions.MXFusionGluonFunction(
SymmetricMatrix(), num_outputs=1, broadcastable=False)
cov = Variable(shape=(self.N, self.K, self.K),
initial_value=mx.nd.broadcast_to(mx.nd.expand_dims(
mx.nd.array(np.eye(self.K, self.K) * 1e-2,
dtype=dtype), 0),
shape=(self.N, self.K,
self.K)))
q.post_cov = sym(cov)
q.post_mean = Variable(shape=(self.N, self.K),
initial_value=mx.nd.array(np.random.randn(
self.N, self.K),
dtype=dtype))
q.z.set_prior(
mf.distributions.MultivariateNormal(mean=q.post_mean,
covariance=q.post_cov))
return q
def test_forward_sampling(self):
dtype = get_default_dtype()
x = np.random.rand(1000, 1)
y = np.random.rand(1000, 1) > 0.5
x_nd, y_nd = mx.nd.array(y, dtype=dtype), mx.nd.array(x, dtype=dtype)
self.net = self.make_net()
self.net(x_nd)
m = self.make_model(self.net)
from mxfusion.inference.meanfield import create_Gaussian_meanfield
from mxfusion.inference import StochasticVariationalInference
from mxfusion.inference.grad_based_inference import GradBasedInference
from mxfusion.inference.batch_loop import BatchInferenceLoop
observed = [m.y, m.x]
q = create_Gaussian_meanfield(model=m, observed=observed)
alg = StochasticVariationalInference(num_samples=3, model=m, posterior=q, observed=observed)
infr = GradBasedInference(inference_algorithm=alg, grad_loop=BatchInferenceLoop())
infr.initialize(y=y_nd, x=x_nd)
infr._verbose = True
infr.run(max_iter=2, learning_rate=1e-2, y=y_nd, x=x_nd)
infr2 = VariationalPosteriorForwardSampling(10, [m.x], infr, [m.r])
infr2.run(x=x_nd)
def make_net(self):
dtype = get_default_dtype()
D = 100
net = nn.HybridSequential(prefix='hybrid0_')
with net.name_scope():
net.add(nn.Dense(D, activation="tanh", dtype=dtype))
net.add(nn.Dense(D, activation="tanh", dtype=dtype))
net.add(nn.Dense(1, flatten=True, dtype=dtype))
net.initialize(mx.init.Xavier(magnitude=3))
return net
def make_net(self):
D = 15
dtype = get_default_dtype()
net = nn.HybridSequential(prefix='hybrid0_')
with net.name_scope():
net.add(nn.Dense(D, activation="tanh", dtype=dtype, in_units=1))
net.add(nn.Dense(D, activation="tanh", dtype=dtype, in_units=D))
net.add(nn.Dense(1, dtype=dtype, in_units=D))
net.initialize(mx.init.Xavier(magnitude=3))
return net
def make_ppca_model(self):
dtype = get_default_dtype()
m = Model()
m.w = Variable(shape=(self.K,self.D), initial_value=mx.nd.array(np.random.randn(self.K,self.D)))
dot = nn.HybridLambda(function='dot')
m.dot = mf.functions.MXFusionGluonFunction(dot, num_outputs=1, broadcastable=False)
cov = mx.nd.broadcast_to(mx.nd.expand_dims(mx.nd.array(np.eye(self.K,self.K), dtype=dtype), 0),shape=(self.N,self.K,self.K))
m.z = mf.distributions.MultivariateNormal.define_variable(mean=mx.nd.zeros(shape=(self.N,self.K), dtype=dtype), covariance=cov, shape=(self.N,self.K))
sigma_2 = Variable(shape=(1,), transformation=PositiveTransformation())
m.x = mf.distributions.Normal.define_variable(mean=m.dot(m.z, m.w), variance=sigma_2, shape=(self.N,self.D))
return m
def test_inference_outcome_passing_success(self):
dtype = get_default_dtype()
observed = [self.m.y, self.m.x]
alg = MAP(model=self.m, observed=observed)
infr = GradBasedInference(inference_algorithm=alg)
infr.run(y=mx.nd.array(np.random.rand(self.D), dtype=dtype),
x=mx.nd.array(np.random.rand(self.D), dtype=dtype),
max_iter=1)
infr2 = VariationalPosteriorForwardSampling(10, [self.m.x], infr,
[self.m.y])
infr2.run(x=mx.nd.array(np.random.rand(self.D), dtype=dtype))
def make_model(self, net):
dtype = get_default_dtype()
m = mf.models.Model(verbose=False)
m.N = mf.components.Variable()
m.f = MXFusionGluonFunction(net, num_outputs=1)
m.x = mf.components.Variable(shape=(m.N, 1))
m.r = m.f(m.x)
for k, v in m.r.factor.parameters.items():
if k.endswith('_weight') or k.endswith('_bias'):
v.set_prior(mf.components.distributions.Normal(mean=mx.nd.array([0], dtype=dtype), variance=mx.nd.array([1e6], dtype=dtype)))
m.y = mf.components.distributions.Categorical.define_variable(log_prob=m.r, num_classes=2, normalization=True, one_hot_encoding=False, shape=(m.N, 1), dtype=dtype)
return m
def test_one_map_example(self):
"""
Tests that the creation of variables from a base gluon block works correctly.
"""
from mxfusion.inference.map import MAP
from mxfusion.inference.grad_based_inference import GradBasedInference
from mxfusion.inference import BatchInferenceLoop
dtype = get_default_dtype()
observed = [self.m.y]
alg = MAP(model=self.m, observed=observed)
infr = GradBasedInference(inference_algorithm=alg,
grad_loop=BatchInferenceLoop())
infr.run(y=mx.nd.array(np.random.rand(10), dtype=dtype), max_iter=10)
def make_bnn_model(self, net):
dtype = get_default_dtype()
m = mf.models.Model(verbose=True)
m.N = mf.components.Variable()
m.f = MXFusionGluonFunction(net, num_outputs=1)
m.x = mf.components.Variable(shape=(m.N,1))
m.v = mf.components.Variable(shape=(1,), transformation=PositiveTransformation(), initial_value=0.01)
m.prior_variance = mf.components.Variable(shape=(1,), transformation=PositiveTransformation())
m.r = m.f(m.x)
for _, v in m.r.factor.parameters.items():
mean = broadcast_to(Variable(mx.nd.array([0], dtype=dtype)),
v.shape)
var = broadcast_to(m.prior_variance, v.shape)
v.set_prior(mf.components.distributions.Normal(mean=mean, variance=var))
m.y = mf.components.distributions.Normal.define_variable(mean=m.r, variance=broadcast_to(m.v, (m.N, 1)), shape=(m.N, 1))
return m
def setUp(self):
dtype = get_default_dtype()
self.D = 10
self.net = nn.HybridSequential()
with self.net.name_scope():
self.net.add(nn.Dense(self.D, activation="relu", dtype=dtype))
self.net.add(nn.Dense(1, activation="relu", dtype=dtype))
self.net.initialize()
from mxnet.gluon import HybridBlock
class DotProduct(HybridBlock):
def hybrid_forward(self, F, x, *args, **kwargs):
return F.dot(x, args[0])
self.mx_dot = DotProduct()
m = mf.models.Model()
m.mean = mf.components.Variable()
m.var = mf.components.Variable(transformation=PositiveTransformation())
m.N = mf.components.Variable()
m.x = mf.components.distributions.Normal.define_variable(
mean=m.mean, variance=m.var, shape=(m.N, ))
m.y = mf.components.distributions.Normal.define_variable(
mean=m.x, variance=mx.nd.array([1], dtype=dtype), shape=(m.N, ))
self.m = m
q = mf.models.posterior.Posterior(m)
q.a = mf.components.Variable()
q.x.set_prior(mf.components.distributions.PointMass(location=q.a))
self.q = q
m = mf.models.Model()
m.mean = mf.components.Variable()
m.var = mf.components.Variable(transformation=PositiveTransformation())
m.N = mf.components.Variable()
m.x = mf.components.distributions.Normal.define_variable(
mean=m.mean, variance=m.var, shape=(m.N, ))
m.y = mf.components.distributions.Normal.define_variable(
mean=m.x, variance=mx.nd.array([1], dtype=dtype), shape=(m.N, ))
self.m2 = m
q = mf.models.posterior.Posterior(m)
q.a = mf.components.Variable()
q.x.set_prior(mf.components.distributions.PointMass(location=q.a))
self.q2 = q
def test_meanfield_save_and_load(self):
dtype = get_default_dtype()
from mxfusion.inference.meanfield import create_Gaussian_meanfield
from mxfusion.inference import StochasticVariationalInference
from mxfusion.inference.grad_based_inference import GradBasedInference
from mxfusion.inference import BatchInferenceLoop
x = np.random.rand(1000, 1)
y = np.random.rand(1000, 1)
x_nd, y_nd = mx.nd.array(y, dtype=dtype), mx.nd.array(x, dtype=dtype)
net = self.make_net()
net(x_nd)
m = self.make_model(net)
observed = [m.y, m.x]
q = create_Gaussian_meanfield(model=m, observed=observed)
alg = StochasticVariationalInference(num_samples=3,
model=m,
observed=observed,
posterior=q)
infr = GradBasedInference(inference_algorithm=alg,
grad_loop=BatchInferenceLoop())
infr.initialize(y=y_nd, x=x_nd)
infr.run(max_iter=1, learning_rate=1e-2, y=y_nd, x=x_nd)
infr.save(self.ZIPNAME)
net2 = self.make_net()
net2(x_nd)
m2 = self.make_model(net2)
observed2 = [m2.y, m2.x]
q2 = create_Gaussian_meanfield(model=m2, observed=observed2)
alg2 = StochasticVariationalInference(num_samples=3,
model=m2,
observed=observed2,
posterior=q2)
infr2 = GradBasedInference(inference_algorithm=alg2,
grad_loop=BatchInferenceLoop())
infr2.initialize(y=y_nd, x=x_nd)
# Load previous parameters
infr2.load(self.ZIPNAME)
for original_uuid, original_param in infr.params.param_dict.items():
original_data = original_param.data().asnumpy()
reloaded_data = infr2.params.param_dict[
infr2._uuid_map[original_uuid]].data().asnumpy()
assert np.all(np.isclose(original_data, reloaded_data))
for original_uuid, original_param in infr.params.constants.items():
if isinstance(original_param, mx.ndarray.ndarray.NDArray):
original_data = original_param.asnumpy()
reloaded_data = infr2.params.constants[
infr2._uuid_map[original_uuid]].asnumpy()
else:
original_data = original_param
reloaded_data = infr2.params.constants[
infr2._uuid_map[original_uuid]]
assert np.all(np.isclose(original_data, reloaded_data))
infr2.run(max_iter=1, learning_rate=1e-2, y=y_nd, x=x_nd)
os.remove(self.ZIPNAME)
