def get_ppca_grad(self, x_train, inf_type, num_samples=100): import random dtype = get_default_dtype() random.seed(0) np.random.seed(0) mx.random.seed(0) m = self.make_ppca_model() q = self.make_ppca_post(m) observed = [m.x] alg = inf_type(num_samples=num_samples, model=m, posterior=q, observed=observed) from mxfusion.inference.grad_based_inference import GradBasedInference from mxfusion.inference import BatchInferenceLoop infr = GradBasedInference(inference_algorithm=alg, grad_loop=BatchInferenceLoop()) infr.initialize(x=mx.nd.array(x_train, dtype=dtype)) infr.run(max_iter=1, learning_rate=1e-2, x=mx.nd.array(x_train, dtype=dtype), verbose=False) return infr, q.post_mean
def test_score_function_rb_minibatch(self): dtype = get_default_dtype() x = np.random.rand(1000, 1) y = np.random.rand(1000, 1) x_nd, y_nd = mx.nd.array(y, dtype=dtype), mx.nd.array(x, dtype=dtype) self.net = self.make_net() self.net(x_nd) m = self.make_bnn_model(self.net) from mxfusion.inference.meanfield import create_Gaussian_meanfield from mxfusion.inference.grad_based_inference import GradBasedInference from mxfusion.inference import MinibatchInferenceLoop observed = [m.y, m.x] q = create_Gaussian_meanfield(model=m, observed=observed) alg = ScoreFunctionRBInference(num_samples=3, model=m, observed=observed, posterior=q) infr = GradBasedInference(inference_algorithm=alg, grad_loop=MinibatchInferenceLoop( batch_size=100, rv_scaling={m.y: 10})) infr.initialize(y=(100, 1), x=(100, 1)) infr.run(max_iter=1, learning_rate=1e-2, y=y_nd, x=x_nd)
def test_meanfield_saving(self): dtype = get_default_dtype() x = np.random.rand(10, 1) y = np.random.rand(10, 1) x_nd, y_nd = mx.nd.array(y, dtype=dtype), mx.nd.array(x, dtype=dtype) self.net = self.make_net() self.net(x_nd) m = self.make_model(self.net) from mxfusion.inference.meanfield import create_Gaussian_meanfield from mxfusion.inference import StochasticVariationalInference from mxfusion.inference.grad_based_inference import GradBasedInference from mxfusion.inference import BatchInferenceLoop observed = [m.y, m.x] q = create_Gaussian_meanfield(model=m, observed=observed) alg = StochasticVariationalInference(num_samples=3, model=m, observed=observed, posterior=q) infr = GradBasedInference(inference_algorithm=alg, grad_loop=BatchInferenceLoop()) infr.initialize(y=y_nd, x=x_nd) infr.run(max_iter=1, learning_rate=1e-2, y=y_nd, x=x_nd) infr.save(self.ZIPNAME) os.remove(self.ZIPNAME)
def make_ppca_post(self, m): from mxfusion.inference import BatchInferenceLoop, GradBasedInference dtype = get_default_dtype() class SymmetricMatrix(mx.gluon.HybridBlock): def hybrid_forward(self, F, x, *args, **kwargs): return F.sum((F.expand_dims(x, 3) * F.expand_dims(x, 2)), axis=-3) q = mf.models.Posterior(m) sym = mf.components.functions.MXFusionGluonFunction( SymmetricMatrix(), num_outputs=1, broadcastable=False) cov = Variable(shape=(self.N, self.K, self.K), initial_value=mx.nd.broadcast_to(mx.nd.expand_dims( mx.nd.array(np.eye(self.K, self.K) * 1e-2, dtype=dtype), 0), shape=(self.N, self.K, self.K))) q.post_cov = sym(cov) q.post_mean = Variable(shape=(self.N, self.K), initial_value=mx.nd.array(np.random.randn( self.N, self.K), dtype=dtype)) q.z.set_prior( mf.distributions.MultivariateNormal(mean=q.post_mean, covariance=q.post_cov)) return q
def test_forward_sampling(self): dtype = get_default_dtype() x = np.random.rand(1000, 1) y = np.random.rand(1000, 1) > 0.5 x_nd, y_nd = mx.nd.array(y, dtype=dtype), mx.nd.array(x, dtype=dtype) self.net = self.make_net() self.net(x_nd) m = self.make_model(self.net) from mxfusion.inference.meanfield import create_Gaussian_meanfield from mxfusion.inference import StochasticVariationalInference from mxfusion.inference.grad_based_inference import GradBasedInference from mxfusion.inference.batch_loop import BatchInferenceLoop observed = [m.y, m.x] q = create_Gaussian_meanfield(model=m, observed=observed) alg = StochasticVariationalInference(num_samples=3, model=m, posterior=q, observed=observed) infr = GradBasedInference(inference_algorithm=alg, grad_loop=BatchInferenceLoop()) infr.initialize(y=y_nd, x=x_nd) infr._verbose = True infr.run(max_iter=2, learning_rate=1e-2, y=y_nd, x=x_nd) infr2 = VariationalPosteriorForwardSampling(10, [m.x], infr, [m.r]) infr2.run(x=x_nd)
def make_net(self): dtype = get_default_dtype() D = 100 net = nn.HybridSequential(prefix='hybrid0_') with net.name_scope(): net.add(nn.Dense(D, activation="tanh", dtype=dtype)) net.add(nn.Dense(D, activation="tanh", dtype=dtype)) net.add(nn.Dense(1, flatten=True, dtype=dtype)) net.initialize(mx.init.Xavier(magnitude=3)) return net
def make_net(self): D = 15 dtype = get_default_dtype() net = nn.HybridSequential(prefix='hybrid0_') with net.name_scope(): net.add(nn.Dense(D, activation="tanh", dtype=dtype, in_units=1)) net.add(nn.Dense(D, activation="tanh", dtype=dtype, in_units=D)) net.add(nn.Dense(1, dtype=dtype, in_units=D)) net.initialize(mx.init.Xavier(magnitude=3)) return net
def make_ppca_model(self): dtype = get_default_dtype() m = Model() m.w = Variable(shape=(self.K,self.D), initial_value=mx.nd.array(np.random.randn(self.K,self.D))) dot = nn.HybridLambda(function='dot') m.dot = mf.functions.MXFusionGluonFunction(dot, num_outputs=1, broadcastable=False) cov = mx.nd.broadcast_to(mx.nd.expand_dims(mx.nd.array(np.eye(self.K,self.K), dtype=dtype), 0),shape=(self.N,self.K,self.K)) m.z = mf.distributions.MultivariateNormal.define_variable(mean=mx.nd.zeros(shape=(self.N,self.K), dtype=dtype), covariance=cov, shape=(self.N,self.K)) sigma_2 = Variable(shape=(1,), transformation=PositiveTransformation()) m.x = mf.distributions.Normal.define_variable(mean=m.dot(m.z, m.w), variance=sigma_2, shape=(self.N,self.D)) return m
def test_inference_outcome_passing_success(self): dtype = get_default_dtype() observed = [self.m.y, self.m.x] alg = MAP(model=self.m, observed=observed) infr = GradBasedInference(inference_algorithm=alg) infr.run(y=mx.nd.array(np.random.rand(self.D), dtype=dtype), x=mx.nd.array(np.random.rand(self.D), dtype=dtype), max_iter=1) infr2 = VariationalPosteriorForwardSampling(10, [self.m.x], infr, [self.m.y]) infr2.run(x=mx.nd.array(np.random.rand(self.D), dtype=dtype))
def make_model(self, net): dtype = get_default_dtype() m = mf.models.Model(verbose=False) m.N = mf.components.Variable() m.f = MXFusionGluonFunction(net, num_outputs=1) m.x = mf.components.Variable(shape=(m.N, 1)) m.r = m.f(m.x) for k, v in m.r.factor.parameters.items(): if k.endswith('_weight') or k.endswith('_bias'): v.set_prior(mf.components.distributions.Normal(mean=mx.nd.array([0], dtype=dtype), variance=mx.nd.array([1e6], dtype=dtype))) m.y = mf.components.distributions.Categorical.define_variable(log_prob=m.r, num_classes=2, normalization=True, one_hot_encoding=False, shape=(m.N, 1), dtype=dtype) return m
def test_one_map_example(self): """ Tests that the creation of variables from a base gluon block works correctly. """ from mxfusion.inference.map import MAP from mxfusion.inference.grad_based_inference import GradBasedInference from mxfusion.inference import BatchInferenceLoop dtype = get_default_dtype() observed = [self.m.y] alg = MAP(model=self.m, observed=observed) infr = GradBasedInference(inference_algorithm=alg, grad_loop=BatchInferenceLoop()) infr.run(y=mx.nd.array(np.random.rand(10), dtype=dtype), max_iter=10)
def make_bnn_model(self, net): dtype = get_default_dtype() m = mf.models.Model(verbose=True) m.N = mf.components.Variable() m.f = MXFusionGluonFunction(net, num_outputs=1) m.x = mf.components.Variable(shape=(m.N,1)) m.v = mf.components.Variable(shape=(1,), transformation=PositiveTransformation(), initial_value=0.01) m.prior_variance = mf.components.Variable(shape=(1,), transformation=PositiveTransformation()) m.r = m.f(m.x) for _, v in m.r.factor.parameters.items(): mean = broadcast_to(Variable(mx.nd.array([0], dtype=dtype)), v.shape) var = broadcast_to(m.prior_variance, v.shape) v.set_prior(mf.components.distributions.Normal(mean=mean, variance=var)) m.y = mf.components.distributions.Normal.define_variable(mean=m.r, variance=broadcast_to(m.v, (m.N, 1)), shape=(m.N, 1)) return m
def setUp(self): dtype = get_default_dtype() self.D = 10 self.net = nn.HybridSequential() with self.net.name_scope(): self.net.add(nn.Dense(self.D, activation="relu", dtype=dtype)) self.net.add(nn.Dense(1, activation="relu", dtype=dtype)) self.net.initialize() from mxnet.gluon import HybridBlock class DotProduct(HybridBlock): def hybrid_forward(self, F, x, *args, **kwargs): return F.dot(x, args[0]) self.mx_dot = DotProduct() m = mf.models.Model() m.mean = mf.components.Variable() m.var = mf.components.Variable(transformation=PositiveTransformation()) m.N = mf.components.Variable() m.x = mf.components.distributions.Normal.define_variable( mean=m.mean, variance=m.var, shape=(m.N, )) m.y = mf.components.distributions.Normal.define_variable( mean=m.x, variance=mx.nd.array([1], dtype=dtype), shape=(m.N, )) self.m = m q = mf.models.posterior.Posterior(m) q.a = mf.components.Variable() q.x.set_prior(mf.components.distributions.PointMass(location=q.a)) self.q = q m = mf.models.Model() m.mean = mf.components.Variable() m.var = mf.components.Variable(transformation=PositiveTransformation()) m.N = mf.components.Variable() m.x = mf.components.distributions.Normal.define_variable( mean=m.mean, variance=m.var, shape=(m.N, )) m.y = mf.components.distributions.Normal.define_variable( mean=m.x, variance=mx.nd.array([1], dtype=dtype), shape=(m.N, )) self.m2 = m q = mf.models.posterior.Posterior(m) q.a = mf.components.Variable() q.x.set_prior(mf.components.distributions.PointMass(location=q.a)) self.q2 = q
def test_meanfield_save_and_load(self): dtype = get_default_dtype() from mxfusion.inference.meanfield import create_Gaussian_meanfield from mxfusion.inference import StochasticVariationalInference from mxfusion.inference.grad_based_inference import GradBasedInference from mxfusion.inference import BatchInferenceLoop x = np.random.rand(1000, 1) y = np.random.rand(1000, 1) x_nd, y_nd = mx.nd.array(y, dtype=dtype), mx.nd.array(x, dtype=dtype) net = self.make_net() net(x_nd) m = self.make_model(net) observed = [m.y, m.x] q = create_Gaussian_meanfield(model=m, observed=observed) alg = StochasticVariationalInference(num_samples=3, model=m, observed=observed, posterior=q) infr = GradBasedInference(inference_algorithm=alg, grad_loop=BatchInferenceLoop()) infr.initialize(y=y_nd, x=x_nd) infr.run(max_iter=1, learning_rate=1e-2, y=y_nd, x=x_nd) infr.save(self.ZIPNAME) net2 = self.make_net() net2(x_nd) m2 = self.make_model(net2) observed2 = [m2.y, m2.x] q2 = create_Gaussian_meanfield(model=m2, observed=observed2) alg2 = StochasticVariationalInference(num_samples=3, model=m2, observed=observed2, posterior=q2) infr2 = GradBasedInference(inference_algorithm=alg2, grad_loop=BatchInferenceLoop()) infr2.initialize(y=y_nd, x=x_nd) # Load previous parameters infr2.load(self.ZIPNAME) for original_uuid, original_param in infr.params.param_dict.items(): original_data = original_param.data().asnumpy() reloaded_data = infr2.params.param_dict[ infr2._uuid_map[original_uuid]].data().asnumpy() assert np.all(np.isclose(original_data, reloaded_data)) for original_uuid, original_param in infr.params.constants.items(): if isinstance(original_param, mx.ndarray.ndarray.NDArray): original_data = original_param.asnumpy() reloaded_data = infr2.params.constants[ infr2._uuid_map[original_uuid]].asnumpy() else: original_data = original_param reloaded_data = infr2.params.constants[ infr2._uuid_map[original_uuid]] assert np.all(np.isclose(original_data, reloaded_data)) infr2.run(max_iter=1, learning_rate=1e-2, y=y_nd, x=x_nd) os.remove(self.ZIPNAME)