def mnist_01(inputs, num_decomps, num_subsets, num_mixtures, num_input_mixtures,
balanced, input_dist, node_type, inf_type, log=False):
# Learning Parameters
additive_smoothing = 100
min_additive_smoothing = 1
# Weight initialization
weight_init_value = tf.initializers.random_uniform(10, 11)
# Generate SPN structure
dense_gen = spn.DenseSPNGenerator(num_decomps=num_decomps,
num_subsets=num_subsets,
num_mixtures=num_mixtures,
input_dist=(spn.DenseSPNGenerator.
InputDist.RAW if input_dist is
"RAW" else spn.
DenseSPNGenerator.
InputDist.MIXTURE),
num_input_mixtures=num_input_mixtures,
balanced=balanced,
node_type=node_type)
root0 = dense_gen.generate(inputs, root_name="root_0")
root1 = dense_gen.generate(inputs, root_name="root_1")
root = spn.Sum(root0, root1, name="root")
spn.generate_weights(root, initializer=weight_init_value)
latent = root.generate_latent_indicators()
# Add EM Learning
additive_smoothing_var = tf.Variable(additive_smoothing, dtype=spn.conf.dtype)
learning = spn.HardEMLearning(root, log=log, value_inference_type=inf_type,
additive_smoothing=additive_smoothing_var)
return root, latent, learning, additive_smoothing, min_additive_smoothing, \
additive_smoothing_var
def test_gradient_on_dense_spn(self, num_decomps, num_subsets,
num_mixtures, input_dist, num_vars,
num_components, softplus):
batch_size = 9
mean_init = np.arange(num_vars * num_components).reshape(
num_vars, num_components)
gl = spn.GaussianLeaf(num_vars=num_vars,
num_components=num_components,
loc_init=mean_init,
softplus_scale=softplus)
gen = spn.DenseSPNGenerator(
num_decomps=num_decomps,
num_subsets=num_subsets,
num_mixtures=num_mixtures,
node_type=spn.DenseSPNGenerator.NodeType.LAYER,
input_dist=input_dist)
root = gen.generate(gl, root_name="root")
with tf.name_scope("Weights"):
spn.generate_weights(root,
tf.initializers.random_uniform(0.0, 1.0),
log=True)
init = spn.initialize_weights(root)
self.assertTrue(root.is_valid())
log_val = root.get_log_value()
spn_grad = spn.Gradient(log=True)
spn_grad.get_gradients(root)
mean_grad_custom, var_grad_custom = gl._compute_gradient(
spn_grad.gradients[gl])
mean_grad_tf, var_grad_tf = tf.gradients(
log_val, [gl.loc_variable, gl.scale_variable])
fd = {gl: np.random.rand(batch_size, num_vars)}
with self.test_session() as sess:
sess.run(init)
mu_grad_custom_val, var_grad_custom_val = sess.run(
[mean_grad_custom, var_grad_custom], fd)
mu_grad_tf_val, var_grad_tf_val = sess.run(
[mean_grad_tf, var_grad_tf], fd)
self.assertAllClose(mu_grad_custom_val,
mu_grad_tf_val,
atol=1e-4,
rtol=1e-4)
self.assertAllClose(var_grad_custom_val,
var_grad_tf_val,
atol=1e-4,
rtol=1e-4)
def mnist_all(inputs,
num_decomps,
num_subsets,
num_mixtures,
num_input_mixtures,
balanced,
input_dist,
node_type,
inf_type,
log=False):
# Learning Parameters
additive_smoothing = 0
min_additive_smoothing = 0
initial_accum_value = 20
# Weight initialization
weight_init_value = tf.initializers.random_uniform(0, 1)
# Add random values before max
add_random = None
use_unweighted = True
# Generate SPN structure
dense_gen = spn.DenseSPNGenerator(
num_decomps=num_decomps,
num_subsets=num_subsets,
num_mixtures=num_mixtures,
input_dist=(spn.DenseSPNGenerator.InputDist.RAW
if input_dist is "RAW" else
spn.DenseSPNGenerator.InputDist.MIXTURE),
num_input_mixtures=num_input_mixtures,
balanced=balanced,
node_type=node_type)
class_roots = [
dense_gen.generate(inputs, root_name=("Class_%d" % i))
for i in range(10)
]
root = spn.Sum(*class_roots, name="root")
spn.generate_weights(root, init_value=weight_init_value)
latent = root.generate_ivs()
# Add EM Learning
additive_smoothing_var = tf.Variable(additive_smoothing,
dtype=spn.conf.dtype)
learning = spn.EMLearning(root,
log=log,
value_inference_type=inf_type,
additive_smoothing=additive_smoothing_var,
add_random=add_random,
initial_accum_value=initial_accum_value,
use_unweighted=use_unweighted)
return root, latent, learning, additive_smoothing, min_additive_smoothing, \
additive_smoothing_var
def generic_dense_test(self, name, num_decomps, num_subsets, num_mixtures,
input_dist, num_input_mixtures):
"""A generic test for DenseSPNGenerator."""
v1 = spn.IVs(num_vars=3, num_vals=2, name="IVs1")
v2 = spn.IVs(num_vars=3, num_vals=2, name="IVs2")
gen = spn.DenseSPNGenerator(num_decomps=num_decomps,
num_subsets=num_subsets,
num_mixtures=num_mixtures,
input_dist=input_dist,
num_input_mixtures=num_input_mixtures)
# Generating SPN
root = gen.generate(v1, v2)
# Generating random weights
with tf.name_scope("Weights"):
spn.generate_weights(root,
tf.initializers.random_uniform(0.0, 1.0))
# Generating weight initializers
init = spn.initialize_weights(root)
# Testing validity
self.assertTrue(root.is_valid())
# Generating value ops
v = root.get_value()
v_log = root.get_log_value()
# Creating session
with self.test_session() as sess:
# Initializing weights
init.run()
# Computing all values
feed = np.array(list(itertools.product(range(2), repeat=6)))
feed_v1 = feed[:, :3]
feed_v2 = feed[:, 3:]
out = sess.run(v, feed_dict={v1: feed_v1, v2: feed_v2})
out_log = sess.run(tf.exp(v_log),
feed_dict={
v1: feed_v1,
v2: feed_v2
})
# Test if partition function is 1.0
self.assertAlmostEqual(out.sum(), 1.0, places=6)
self.assertAlmostEqual(out_log.sum(), 1.0, places=6)
self.write_tf_graph(sess, self.sid(), self.cid())
def test_generte_set_errors(self):
"""Detecting structure errors in __generate_set"""
gen = spn.DenseSPNGenerator(num_decomps=2,
num_subsets=3,
num_mixtures=2)
v1 = spn.IVs(num_vars=2, num_vals=4)
v2 = spn.ContVars(num_vars=3, name="ContVars1")
v3 = spn.ContVars(num_vars=2, name="ContVars2")
s1 = spn.Sum(v3, v2)
n1 = spn.Concat(v2)
with self.assertRaises(spn.StructureError):
gen._DenseSPNGenerator__generate_set([
spn.Input(v1, [0, 3, 2, 6, 7]),
spn.Input(v2, [1, 2]),
spn.Input(s1, None),
spn.Input(n1, None)
])
def dense_block(inputs,
num_decomps,
num_subsets,
num_mixtures,
num_input_mixtures,
balanced,
input_dist,
inf_type,
log=False):
# Set node-type as single-node
node_type = spn.DenseSPNGenerator.NodeType.BLOCK
# Create a dense generator
gen = spn.DenseSPNGenerator(
num_decomps=num_decomps,
num_subsets=num_subsets,
num_mixtures=num_mixtures,
num_input_mixtures=num_input_mixtures,
balanced=balanced,
node_type=node_type,
input_dist=(spn.DenseSPNGenerator.InputDist.RAW
if input_dist is "RAW" else
spn.DenseSPNGenerator.InputDist.MIXTURE))
# Generate a dense SPN, with single-op nodes, and all weights in the network
root = gen.generate(inputs, root_name="root")
spn.generate_weights(root, tf.initializers.random_uniform(0.0, 1.0))
# Generate path ops based on inf_type and log
if log:
mpe_path_gen = spn.MPEPath(value_inference_type=inf_type, log=True)
else:
mpe_path_gen = spn.MPEPath(value_inference_type=inf_type,
log=False)
mpe_path_gen.get_mpe_path(root)
path_ops = [
mpe_path_gen.counts[inp]
for inp in (inputs if isinstance(inputs, list) else [inputs])
]
return root, spn.initialize_weights(root), path_ops
def test_traversing_on_dense(self):
"""Compare traversal algs on dense SPN"""
def fun1(node, *args):
counter[0] += 1
def fun2(node, *args):
counter[0] += 1
if node.is_op:
return [None] * len(node.inputs)
# Generate dense graph
v1 = spn.IndicatorLeaf(num_vars=3, num_vals=2, name="IndicatorLeaf1")
v2 = spn.IndicatorLeaf(num_vars=3, num_vals=2, name="IndicatorLeaf2")
gen = spn.DenseSPNGenerator(num_decomps=2,
num_subsets=3,
num_mixtures=2,
input_dist=spn.DenseSPNGenerator.InputDist.MIXTURE,
num_input_mixtures=None)
root = gen.generate(v1, v2)
spn.generate_weights(root)
# Run traversal algs and count nodes
counter = [0]
spn.compute_graph_up_down(root, down_fun=fun2, graph_input=1)
c1 = counter[0]
counter = [0]
spn.compute_graph_up(root, val_fun=fun1)
c2 = counter[0]
counter = [0]
spn.traverse_graph(root, fun=fun1, skip_params=False)
c3 = counter[0]
# Compare
self.assertEqual(c1, c3)
self.assertEqual(c2, c3)
def test_generte_set(self):
"""Generation of sets of inputs with __generate_set"""
gen = spn.DenseSPNGenerator(num_decomps=2,
num_subsets=3,
num_mixtures=2)
v1 = spn.IVs(num_vars=2, num_vals=4)
v2 = spn.ContVars(num_vars=3, name="ContVars1")
v3 = spn.ContVars(num_vars=2, name="ContVars2")
s1 = spn.Sum(v3)
n1 = spn.Concat(v2)
out = gen._DenseSPNGenerator__generate_set([
spn.Input(v1, [0, 3, 2, 6, 7]),
spn.Input(v2, [1, 2]),
spn.Input(s1, None),
spn.Input(n1, None)
])
# scope_dict:
# Scope({IVs(0x7f00cb4049b0):0}): {(IVs(0x7f00cb4049b0), 0),
# (IVs(0x7f00cb4049b0), 2),
# (IVs(0x7f00cb4049b0), 3)},
# Scope({IVs(0x7f00cb4049b0):1}): {(IVs(0x7f00cb4049b0), 7),
# (IVs(0x7f00cb4049b0), 6)},
# Scope({ContVars1(0x7f00b7982ef0):1}): {(Concat(0x7f00cb404d68), 1),
# (ContVars1(0x7f00b7982ef0), 1)},
# Scope({ContVars1(0x7f00b7982ef0):2}): {(Concat(0x7f00cb404d68), 2),
# (ContVars1(0x7f00b7982ef0), 2)},
# Scope({ContVars1(0x7f00b7982ef0):0}): {(Concat(0x7f00cb404d68), 0)},
# Scope({ContVars2(0x7f00cb391eb8):0, ContVars2(0x7f00cb391eb8):1}): {
# (Sum(0x7f00cb404a90), 0)}}
# Since order is undetermined, we check items
self.assertEqual(len(out), 6)
self.assertIn(tuple(sorted([(v2, 1), (n1, 1)])), out)
self.assertIn(tuple(sorted([(v2, 2), (n1, 2)])), out)
self.assertIn(tuple(sorted([(n1, 0)])), out)
self.assertIn(tuple(sorted([(v1, 0), (v1, 2), (v1, 3)])), out)
self.assertIn(tuple(sorted([(v1, 6), (v1, 7)])), out)
self.assertIn(tuple(sorted([(s1, 0)])), out)
def dense_block(inputs,
num_decomps,
num_subsets,
num_mixtures,
num_input_mixtures,
balanced,
input_dist,
inf_type,
log=False):
# Set node-type as single-node
node_type = spn.DenseSPNGenerator.NodeType.BLOCK
# Create a dense generator
gen = spn.DenseSPNGenerator(
num_decomps=num_decomps,
num_subsets=num_subsets,
num_mixtures=num_mixtures,
num_input_mixtures=num_input_mixtures,
balanced=balanced,
node_type=node_type,
input_dist=(spn.DenseSPNGenerator.InputDist.RAW
if input_dist is "RAW" else
spn.DenseSPNGenerator.InputDist.MIXTURE))
# Generate a dense SPN, with block-nodes, and all weights in the network
root = gen.generate(inputs, root_name="root")
spn.generate_weights(root, tf.initializers.random_uniform(0.0, 1.0))
# Generate value ops based on inf_type and log
if log:
value_op = root.get_log_value(inference_type=inf_type)
else:
value_op = root.get_value(inference_type=inf_type)
return root, spn.initialize_weights(root), value_op
def test_generte_set(self):
"""Generation of sets of inputs with __generate_set"""
gen = spn.DenseSPNGenerator(num_decomps=2,
num_subsets=3,
num_mixtures=2)
v1 = spn.IVs(num_vars=2, num_vals=4)
v2 = spn.ContVars(num_vars=3, name="ContVars1")
v3 = spn.ContVars(num_vars=2, name="ContVars2")
s1 = spn.Sum(v3)
n1 = spn.Concat(v2)
out = gen._DenseSPNGenerator__generate_set([
spn.Input(v1, [0, 3, 2, 6, 7]),
spn.Input(v2, [1, 2]),
spn.Input(s1, None),
spn.Input(n1, None)
])
# Since order is undetermined, we check items
self.assertEqual(len(out), 6)
self.assertIn(tuple(sorted([(v2, 1), (n1, 1)])), out)
self.assertIn(tuple(sorted([(v2, 2), (n1, 2)])), out)
self.assertIn(tuple(sorted([(n1, 0)])), out)
self.assertIn(tuple(sorted([(v1, 0), (v1, 2), (v1, 3)])), out)
self.assertIn(tuple(sorted([(v1, 6), (v1, 7)])), out)
self.assertIn(tuple(sorted([(s1, 0)])), out)
def test_generate_spn(self, num_decomps, num_subsets, num_mixtures,
num_input_mixtures, input_dims, input_dist, balanced,
node_type, log_weights):
"""A generic test for DenseSPNGenerator."""
if input_dist == spn.DenseSPNGenerator.InputDist.RAW \
and num_input_mixtures != 1:
# Redundant test case, so just return
return
# Input parameters
num_inputs = input_dims[0]
num_vars = input_dims[1]
num_vals = 2
printc("\n- num_inputs: %s" % num_inputs)
printc("- num_vars: %s" % num_vars)
printc("- num_vals: %s" % num_vals)
printc("- num_decomps: %s" % num_decomps)
printc("- num_subsets: %s" % num_subsets)
printc("- num_mixtures: %s" % num_mixtures)
printc("- input_dist: %s" %
("MIXTURE" if input_dist
== spn.DenseSPNGenerator.InputDist.MIXTURE else "RAW"))
printc("- balanced: %s" % balanced)
printc("- num_input_mixtures: %s" % num_input_mixtures)
printc("- node_type: %s" %
("SINGLE" if node_type == spn.DenseSPNGenerator.NodeType.SINGLE
else "BLOCK" if node_type
== spn.DenseSPNGenerator.NodeType.BLOCK else "LAYER"))
printc("- log_weights: %s" % log_weights)
# Inputs
inputs = [
spn.IVs(num_vars=num_vars,
num_vals=num_vals,
name=("IVs_%d" % (i + 1))) for i in range(num_inputs)
]
gen = spn.DenseSPNGenerator(num_decomps=num_decomps,
num_subsets=num_subsets,
num_mixtures=num_mixtures,
input_dist=input_dist,
balanced=balanced,
num_input_mixtures=num_input_mixtures,
node_type=node_type)
# Generate Sub-SPNs
sub_spns = [
gen.generate(*inputs, root_name=("sub_root_%d" % (i + 1)))
for i in range(3)
]
# Generate random weights for the first sub-SPN
with tf.name_scope("Weights"):
spn.generate_weights(sub_spns[0],
tf.initializers.random_uniform(0.0, 1.0),
log=log_weights)
# Initialize weights of the first sub-SPN
sub_spn_init = spn.initialize_weights(sub_spns[0])
# Testing validity of the first sub-SPN
self.assertTrue(sub_spns[0].is_valid())
# Generate value ops of the first sub-SPN
sub_spn_v = sub_spns[0].get_value()
sub_spn_v_log = sub_spns[0].get_log_value()
# Generate path ops of the first sub-SPN
sub_spn_mpe_path_gen = spn.MPEPath(log=False)
sub_spn_mpe_path_gen_log = spn.MPEPath(log=True)
sub_spn_mpe_path_gen.get_mpe_path(sub_spns[0])
sub_spn_mpe_path_gen_log.get_mpe_path(sub_spns[0])
sub_spn_path = [sub_spn_mpe_path_gen.counts[inp] for inp in inputs]
sub_spn_path_log = [
sub_spn_mpe_path_gen_log.counts[inp] for inp in inputs
]
# Collect all weight nodes of the first sub-SPN
sub_spn_weight_nodes = []
def fun(node):
if node.is_param:
sub_spn_weight_nodes.append(node)
spn.traverse_graph(sub_spns[0], fun=fun)
# Generate an upper-SPN over sub-SPNs
products_lower = []
for sub_spn in sub_spns:
products_lower.append([v.node for v in sub_spn.values])
num_top_mixtures = [2, 1, 3]
sums_lower = []
for prods, num_top_mix in zip(products_lower, num_top_mixtures):
if node_type == spn.DenseSPNGenerator.NodeType.SINGLE:
sums_lower.append(
[spn.Sum(*prods) for _ in range(num_top_mix)])
elif node_type == spn.DenseSPNGenerator.NodeType.BLOCK:
sums_lower.append([spn.ParSums(*prods, num_sums=num_top_mix)])
else:
sums_lower.append([
spn.SumsLayer(*prods * num_top_mix,
num_or_size_sums=num_top_mix)
])
# Generate upper-SPN
root = gen.generate(*list(itertools.chain(*sums_lower)),
root_name="root")
# Generate random weights for the SPN
with tf.name_scope("Weights"):
spn.generate_weights(root,
tf.initializers.random_uniform(0.0, 1.0),
log=log_weights)
# Initialize weight of the SPN
spn_init = spn.initialize_weights(root)
# Testing validity of the SPN
self.assertTrue(root.is_valid())
# Generate value ops of the SPN
spn_v = root.get_value()
spn_v_log = root.get_log_value()
# Generate path ops of the SPN
spn_mpe_path_gen = spn.MPEPath(log=False)
spn_mpe_path_gen_log = spn.MPEPath(log=True)
spn_mpe_path_gen.get_mpe_path(root)
spn_mpe_path_gen_log.get_mpe_path(root)
spn_path = [spn_mpe_path_gen.counts[inp] for inp in inputs]
spn_path_log = [spn_mpe_path_gen_log.counts[inp] for inp in inputs]
# Collect all weight nodes in the SPN
spn_weight_nodes = []
def fun(node):
if node.is_param:
spn_weight_nodes.append(node)
spn.traverse_graph(root, fun=fun)
# Create a session
with self.test_session() as sess:
# Initializing weights
sess.run(sub_spn_init)
sess.run(spn_init)
# Generate input feed
feed = np.array(
list(
itertools.product(range(num_vals),
repeat=(num_inputs * num_vars))))
batch_size = feed.shape[0]
feed_dict = {}
for inp, f in zip(inputs, np.split(feed, num_inputs, axis=1)):
feed_dict[inp] = f
# Compute all values and paths of sub-SPN
sub_spn_out = sess.run(sub_spn_v, feed_dict=feed_dict)
sub_spn_out_log = sess.run(tf.exp(sub_spn_v_log),
feed_dict=feed_dict)
sub_spn_out_path = sess.run(sub_spn_path, feed_dict=feed_dict)
sub_spn_out_path_log = sess.run(sub_spn_path_log,
feed_dict=feed_dict)
# Compute all values and paths of the complete SPN
spn_out = sess.run(spn_v, feed_dict=feed_dict)
spn_out_log = sess.run(tf.exp(spn_v_log), feed_dict=feed_dict)
spn_out_path = sess.run(spn_path, feed_dict=feed_dict)
spn_out_path_log = sess.run(spn_path_log, feed_dict=feed_dict)
# Test if partition function of the sub-SPN and of the
# complete SPN is 1.0
self.assertAlmostEqual(sub_spn_out.sum(), 1.0, places=6)
self.assertAlmostEqual(sub_spn_out_log.sum(), 1.0, places=6)
self.assertAlmostEqual(spn_out.sum(), 1.0, places=6)
self.assertAlmostEqual(spn_out_log.sum(), 1.0, places=6)
# Test if the sum of counts for each value of each variable
# (6 variables, with 2 values each) = batch-size / num-vals
self.assertEqual(
np.sum(np.hstack(sub_spn_out_path), axis=0).tolist(),
[batch_size // num_vals] * num_inputs * num_vars * num_vals)
self.assertEqual(
np.sum(np.hstack(sub_spn_out_path_log), axis=0).tolist(),
[batch_size // num_vals] * num_inputs * num_vars * num_vals)
self.assertEqual(
np.sum(np.hstack(spn_out_path), axis=0).tolist(),
[batch_size // num_vals] * num_inputs * num_vars * num_vals)
self.assertEqual(
np.sum(np.hstack(spn_out_path_log), axis=0).tolist(),
[batch_size // num_vals] * num_inputs * num_vars * num_vals)
