def test_sum_node_is_complete():
# create a sum node with a scope
scope = frozenset({0, 2, 7, 13})
sum_node = SumNode(var_scope=scope)
# creating children with same scope
children = [ProductNode(var_scope=scope) for i in range(4)]
for prod_node in children:
sum_node.add_child(prod_node, 1.0)
assert sum_node.is_complete()
# now altering one child's scope with one less var
children[0].var_scope = frozenset({0, 7, 13})
assert sum_node.is_complete() is False
# now adding one more
children[0].var_scope = scope
children[3].var_scope = frozenset({0, 2, 7, 13, 3})
assert not sum_node.is_complete()
# now checking with indicator input nodes
var = 4
sum_node = SumNode(var_scope=frozenset({var}))
children = [CategoricalIndicatorNode(var=var, var_val=i)
for i in range(4)]
for input_node in children:
sum_node.add_child(input_node, 1.0)
assert sum_node.is_complete()
def test_spn_set_get_weights():
# create a simple spn
root_node = SumNode()
root_layer = SumLayer([root_node])
prod_node_1 = ProductNode()
prod_node_2 = ProductNode()
root_node.add_child(prod_node_1, 0.5)
root_node.add_child(prod_node_2, 0.5)
prod_layer = ProductLayer([prod_node_1, prod_node_2])
sum_node_1 = SumNode()
sum_node_2 = SumNode()
sum_node_3 = SumNode()
prod_node_1.add_child(sum_node_1)
prod_node_1.add_child(sum_node_2)
prod_node_2.add_child(sum_node_2)
prod_node_2.add_child(sum_node_3)
sum_layer = SumLayer([sum_node_1, sum_node_2, sum_node_3])
ind_node_1 = CategoricalIndicatorNode(var=0, var_val=1)
ind_node_2 = CategoricalIndicatorNode(var=0, var_val=1)
ind_node_3 = CategoricalIndicatorNode(var=0, var_val=1)
ind_node_4 = CategoricalIndicatorNode(var=0, var_val=1)
ind_node_5 = CategoricalIndicatorNode(var=0, var_val=1)
input_layer = CategoricalInputLayer(
nodes=[ind_node_1, ind_node_2, ind_node_3, ind_node_4, ind_node_5])
sum_node_1.add_child(ind_node_1, 0.2)
sum_node_1.add_child(ind_node_2, 0.2)
sum_node_2.add_child(ind_node_2, 0.2)
sum_node_2.add_child(ind_node_3, 0.2)
sum_node_2.add_child(ind_node_4, 0.2)
sum_node_3.add_child(ind_node_4, 0.2)
sum_node_3.add_child(ind_node_5, 0.2)
spn = Spn(input_layer=input_layer,
layers=[sum_layer, prod_layer, root_layer])
print(spn)
# storing these weights
curr_weights = spn.get_weights()
# setting the new weights
spn.set_weights(weights_ds)
# getting them again
new_weights = spn.get_weights()
# comparing them
assert new_weights == weights_ds
# now setting back the previous one
spn.set_weights(curr_weights)
# getting them back again
old_weights = spn.get_weights()
# and checking
assert old_weights == curr_weights
def test_sum_node_create_and_eval():
# create child nodes
child1 = Node()
val1 = 1.
child1.set_val(val1)
child2 = Node()
val2 = 1.
child2.set_val(val2)
# create sum node and adding children to it
sum_node = SumNode()
weight1 = 0.8
weight2 = 0.2
sum_node.add_child(child1, weight1)
sum_node.add_child(child2, weight2)
assert len(sum_node.children) == 2
assert len(sum_node.weights) == 2
assert len(sum_node.log_weights) == 2
log_weights = [log(weight1), log(weight2)]
assert log_weights == sum_node.log_weights
print(sum_node)
# evaluating
sum_node.eval()
print(sum_node.log_val)
assert_almost_equal(sum_node.log_val,
log(val1 * weight1 + val2 * weight2),
places=15)
# changing values 1,0
val1 = 1.
child1.set_val(val1)
val2 = 0.
child2.set_val(val2)
# evaluating
sum_node.eval()
print(sum_node.log_val)
assert_almost_equal(sum_node.log_val,
log(val1 * weight1 + val2 * weight2),
places=15)
# changing values 0,0 -> LOG_ZERO
val1 = 0.
child1.set_val(val1)
val2 = 0.
child2.set_val(val2)
# evaluating
sum_node.eval()
print(sum_node.log_val)
assert_almost_equal(sum_node.log_val,
LOG_ZERO,
places=15)
def test_sum_node_create_and_eval_keras():
n_trials = 100
for i in range(n_trials):
n_children = numpy.random.randint(1, 100)
print('n children', n_children)
children = [Node() for c in range(n_children)]
weights = numpy.random.rand(n_children)
weights = weights / weights.sum()
#
# create sum node and adding children to it
sum_node = SumNode()
for child, w in zip(children, weights):
sum_node.add_child(child, w)
child.log_vals = K.placeholder(ndim=2)
assert len(sum_node.children) == n_children
assert len(sum_node.weights) == n_children
assert len(sum_node.log_weights) == n_children
print(sum_node)
#
# evaluating for fake probabilities
n_instances = numpy.random.randint(1, 100)
print('n instances', n_instances)
probs = numpy.random.rand(n_instances, n_children) # .astype(theano.config.floatX)
log_probs = numpy.log(probs)
log_vals = []
for d in range(n_instances):
for c, child in enumerate(children):
child.set_val(probs[d, c])
sum_node.eval()
print('sum node eval')
print(sum_node.log_val)
log_vals.append(sum_node.log_val)
#
# now theano
sum_node.build_k()
eval_sum_node_f = K.function(inputs=[c.log_vals for c in children],
outputs=[sum_node.log_vals])
keras_log_vals = eval_sum_node_f([log_probs[:, c].reshape(log_probs.shape[0], 1)
for c in range(n_children)])[0]
print(keras_log_vals)
assert_array_almost_equal(numpy.array(log_vals).reshape(log_probs.shape[0], 1),
keras_log_vals,
decimal=4)
def build_spn_layers(input_layer):
# this is ugly... TODO try to beutify this process
ind1 = input_layer._nodes[0]
ind2 = input_layer._nodes[1]
ind3 = input_layer._nodes[2]
ind4 = input_layer._nodes[3]
ind5 = input_layer._nodes[4]
ind6 = input_layer._nodes[5]
ind7 = input_layer._nodes[6]
ind8 = input_layer._nodes[7]
# creating sum nodes
sum_node1 = SumNode()
sum_node2 = SumNode()
sum_node3 = SumNode()
sum_node4 = SumNode()
# linking them with nodes
sum_node1.add_child(ind1, 0.5)
sum_node1.add_child(ind2, 0.5)
sum_node2.add_child(ind3, 0.1)
sum_node2.add_child(ind4, 0.9)
sum_node3.add_child(ind5, 0.3)
sum_node3.add_child(ind6, 0.7)
sum_node4.add_child(ind7, 0.6)
sum_node4.add_child(ind8, 0.4)
# creating sumlayer
sum_layer = SumLayer([sum_node1,
sum_node2,
sum_node3,
sum_node4])
# creating product nodes
prod_node1 = ProductNode()
prod_node2 = ProductNode()
prod_node3 = ProductNode()
# linking them to sum nodes
prod_node1.add_child(sum_node1)
prod_node1.add_child(sum_node2)
prod_node2.add_child(sum_node2)
prod_node2.add_child(sum_node3)
prod_node3.add_child(sum_node3)
prod_node3.add_child(sum_node4)
# creating a product layer
prod_layer = ProductLayer([prod_node1,
prod_node2,
prod_node3])
return sum_layer, prod_layer
def test_sum_node_normalize():
# create child nodes
child1 = Node()
val1 = 1.
child1.set_val(val1)
child2 = Node()
val2 = 1.
child2.set_val(val2)
# create sum node and adding children to it
sum_node = SumNode()
weight1 = 1.
weight2 = 0.2
weights = [weight1, weight2]
sum_node.add_child(child1, weight1)
sum_node.add_child(child2, weight2)
un_sum = sum(weights)
# normalizing
sum_node.normalize()
assert len(sum_node.children) == 2
assert len(sum_node.weights) == 2
assert len(sum_node.log_weights) == 2
# checking weight sum
w_sum = sum(sum_node.weights)
assert w_sum == 1.
# and check the correct values
normal_sum = [weight / un_sum for weight in weights]
print(normal_sum)
assert normal_sum == sum_node.weights
# checking log_weights
log_weights = [log(weight) for weight in normal_sum]
print(log_weights)
assert log_weights == sum_node.log_weights
def test_sum_layer_is_complete():
# creating two scopes and two sum nodes
scope1 = frozenset({0, 2, 3})
scope2 = frozenset({10})
sum_node_1 = SumNode(var_scope=scope1)
sum_node_2 = SumNode(var_scope=scope2)
# adding product nodes as children to the first, indicator the second
for i in range(4):
sum_node_1.add_child(ProductNode(var_scope=scope1), 1.0)
sum_node_2.add_child(CategoricalIndicatorNode(var=10, var_val=i), 1.0)
# creating sum layer
sum_layer = SumLayer(nodes=[sum_node_1, sum_node_2])
assert sum_layer.is_complete()
# now with errors in scope
scope3 = frozenset({6})
sum_node_1 = SumNode(var_scope=scope1)
sum_node_2 = SumNode(var_scope=scope3)
# adding product nodes as children to the first, indicator the second
for i in range(4):
sum_node_1.add_child(ProductNode(var_scope=scope1), 1.0)
sum_node_2.add_child(CategoricalIndicatorNode(var=10, var_val=i), 1.0)
# creating sum layer
sum_layer = SumLayer(nodes=[sum_node_1, sum_node_2])
assert not sum_layer.is_complete()
sum_node_2.var_scope = scope2
assert sum_layer.is_complete()
sum_node_2.children[3].var_scope = scope3
assert not sum_layer.is_complete()
def test_linked_to_theano_indicator():
# creating single nodes
root = SumNode()
prod1 = ProductNode()
prod2 = ProductNode()
prod3 = ProductNode()
sum1 = SumNode()
sum2 = SumNode()
sum3 = SumNode()
sum4 = SumNode()
ind1 = CategoricalIndicatorNode(var=0, var_val=0)
ind2 = CategoricalIndicatorNode(var=0, var_val=1)
ind3 = CategoricalIndicatorNode(var=1, var_val=0)
ind4 = CategoricalIndicatorNode(var=1, var_val=1)
ind5 = CategoricalIndicatorNode(var=2, var_val=0)
ind6 = CategoricalIndicatorNode(var=2, var_val=1)
ind7 = CategoricalIndicatorNode(var=2, var_val=2)
ind8 = CategoricalIndicatorNode(var=3, var_val=0)
ind9 = CategoricalIndicatorNode(var=3, var_val=1)
ind10 = CategoricalIndicatorNode(var=3, var_val=2)
ind11 = CategoricalIndicatorNode(var=3, var_val=3)
prod4 = ProductNode()
prod5 = ProductNode()
prod6 = ProductNode()
prod7 = ProductNode()
# linking nodes
root.add_child(prod1, 0.3)
root. add_child(prod2, 0.3)
root.add_child(prod3, 0.4)
prod1.add_child(sum1)
prod1.add_child(sum2)
prod2.add_child(ind7)
prod2.add_child(ind8)
prod2.add_child(ind11)
prod3.add_child(sum3)
prod3.add_child(sum4)
sum1.add_child(ind1, 0.3)
sum1.add_child(ind2, 0.3)
sum1.add_child(prod4, 0.4)
sum2.add_child(ind2, 0.5)
sum2.add_child(prod4, 0.2)
sum2.add_child(prod5, 0.3)
sum3.add_child(prod6, 0.5)
sum3.add_child(prod7, 0.5)
sum4.add_child(prod6, 0.5)
sum4.add_child(prod7, 0.5)
prod4.add_child(ind3)
prod4.add_child(ind4)
prod5.add_child(ind5)
prod5.add_child(ind6)
prod6.add_child(ind9)
prod6.add_child(ind10)
prod7.add_child(ind9)
prod7.add_child(ind10)
# building layers from nodes
root_layer = SumLayerLinked([root])
prod_layer = ProductLayerLinked([prod1, prod2, prod3])
sum_layer = SumLayerLinked([sum1, sum2, sum3, sum4])
aprod_layer = ProductLayerLinked([prod4, prod5, prod6, prod7])
ind_layer = CategoricalIndicatorLayer(nodes=[ind1, ind2,
ind3, ind4,
ind5, ind6,
ind7, ind8,
ind9, ind10,
ind11])
# creating the linked spn
spn_linked = SpnLinked(input_layer=ind_layer,
layers=[aprod_layer,
sum_layer,
prod_layer,
root_layer])
print(spn_linked)
# converting to theano repr
spn_theano = SpnFactory.linked_to_theano(spn_linked)
print(spn_theano)
# time for some inference comparison
for instance in I:
print('linked')
res_l = spn_linked.eval(instance)
print(res_l)
print('theano')
res_t = spn_theano.eval(instance)
print(res_t)
assert_array_almost_equal(res_l, res_t)
def test_spn_mpe_eval_and_traversal():
# create initial layer
node1 = Node()
node2 = Node()
node3 = Node()
node4 = Node()
node5 = Node()
input_layer = CategoricalInputLayer([node1, node2,
node3, node4,
node5])
# top layer made by 3 sum nodes
sum1 = SumNode()
sum2 = SumNode()
sum3 = SumNode()
# linking to input nodes
weight11 = 0.3
sum1.add_child(node1, weight11)
weight12 = 0.3
sum1.add_child(node2, weight12)
weight13 = 0.4
sum1.add_child(node3, weight13)
weight22 = 0.15
sum2.add_child(node2, weight22)
weight23 = 0.15
sum2.add_child(node3, weight23)
weight24 = 0.7
sum2.add_child(node4, weight24)
weight33 = 0.4
sum3.add_child(node3, weight33)
weight34 = 0.25
sum3.add_child(node4, weight34)
weight35 = 0.35
sum3.add_child(node5, weight35)
sum_layer = SumLayer([sum1, sum2, sum3])
# another layer with two product nodes
prod1 = ProductNode()
prod2 = ProductNode()
prod1.add_child(sum1)
prod1.add_child(sum2)
prod2.add_child(sum2)
prod2.add_child(sum3)
prod_layer = ProductLayer([prod1, prod2])
# root layer, double sum
root1 = SumNode()
root2 = SumNode()
weightr11 = 0.5
root1.add_child(prod1, weightr11)
weightr12 = 0.5
root1.add_child(prod2, weightr12)
weightr21 = 0.9
root2.add_child(prod1, weightr21)
weightr22 = 0.1
root2.add_child(prod2, weightr22)
root_layer = SumLayer([root1, root2])
# create the spn
spn = Spn(input_layer=input_layer,
layers=[sum_layer, prod_layer, root_layer])
print('===================')
print(spn)
print('===================')
# setting the input values
val1 = 0.0
node1.set_val(val1)
val2 = 0.5
node2.set_val(val2)
val3 = 0.3
node3.set_val(val3)
val4 = 1.0
node4.set_val(val4)
val5 = 0.0
node5.set_val(val5)
# evaluating the spn with MPE inference
res = spn.test_mpe_eval()
print('spn eval\'d', res)
# testing it
#
# testing the max layer
max1 = max(val1 * weight11,
val2 * weight12,
val3 * weight13)
max2 = max(val2 * weight22,
val3 * weight23,
val4 * weight24)
max3 = max(val3 * weight33,
val4 * weight34,
val5 * weight35)
log_max1 = log(max1) if not numpy.isclose(max1, 0) else LOG_ZERO
log_max2 = log(max2) if not numpy.isclose(max2, 0) else LOG_ZERO
log_max3 = log(max3) if not numpy.isclose(max3, 0) else LOG_ZERO
print('expected max vals {0}, {1}, {2}'.format(log_max1,
log_max2,
log_max3))
print('found max vals {0}, {1}, {2}'.format(sum1.log_val,
sum2.log_val,
sum3.log_val))
if IS_LOG_ZERO(log_max1):
assert IS_LOG_ZERO(sum1.log_val)
else:
assert_almost_equal(log_max1, sum1.log_val)
if IS_LOG_ZERO(log_max2):
assert IS_LOG_ZERO(sum2.log_val)
else:
assert_almost_equal(log_max2, sum2.log_val)
if IS_LOG_ZERO(log_max3):
assert IS_LOG_ZERO(sum3.log_val)
else:
assert_almost_equal(log_max3, sum3.log_val)
# product layer is assumed to be fine, but let's check
# it anyways
prod_val1 = max1 * max2
prod_val2 = max2 * max3
prod_log_val1 = log_max1 + log_max2
prod_log_val2 = log_max2 + log_max3
print('exp prod vals {0}, {1}'.format(prod_log_val1,
prod_log_val2))
print('rea prod vals {0}, {1}'.format(prod1.log_val,
prod2.log_val))
if IS_LOG_ZERO(prod_log_val1):
assert IS_LOG_ZERO(prod1.log_val)
else:
assert_almost_equal(prod_log_val1, prod1.log_val)
if IS_LOG_ZERO(prod_log_val2):
assert IS_LOG_ZERO(prod2.log_val)
else:
assert_almost_equal(prod_log_val2, prod2.log_val)
# root layer, again a sum layer
root_val1 = max(prod_val1 * weightr11,
prod_val2 * weightr12)
root_val2 = max(prod_val1 * weightr21,
prod_val2 * weightr22)
root_log_val1 = log(root_val1) if not numpy.isclose(
root_val1, 0) else LOG_ZERO
root_log_val2 = log(root_val2) if not numpy.isclose(
root_val2, 0) else LOG_ZERO
print('exp root vals {0}, {1}'.format(root_log_val1,
root_log_val2))
print('found ro vals {0}, {1}'.format(root1.log_val,
root2.log_val))
if IS_LOG_ZERO(root_log_val1):
assert IS_LOG_ZERO(root1.log_val)
else:
assert_almost_equal(root_log_val1, root1.log_val)
if IS_LOG_ZERO(root_log_val2):
assert IS_LOG_ZERO(root2.log_val)
else:
assert_almost_equal(root_log_val2, root2.log_val)
# now we are traversing top down the net
print('mpe traversing')
for i, j, k in spn.mpe_traversal():
print(i, j, k)
def create_valid_toy_spn():
# root layer
whole_scope = frozenset({0, 1, 2, 3})
root_node = SumNode(var_scope=whole_scope)
root_layer = SumLayer([root_node])
# prod layer
prod_node_1 = ProductNode(var_scope=whole_scope)
prod_node_2 = ProductNode(var_scope=whole_scope)
prod_layer_1 = ProductLayer([prod_node_1, prod_node_2])
root_node.add_child(prod_node_1, 0.5)
root_node.add_child(prod_node_2, 0.5)
# sum layer
scope_1 = frozenset({0, 1})
scope_2 = frozenset({2})
scope_3 = frozenset({3})
scope_4 = frozenset({2, 3})
sum_node_1 = SumNode(var_scope=scope_1)
sum_node_2 = SumNode(var_scope=scope_2)
sum_node_3 = SumNode(var_scope=scope_3)
sum_node_4 = SumNode(var_scope=scope_4)
prod_node_1.add_child(sum_node_1)
prod_node_1.add_child(sum_node_2)
prod_node_1.add_child(sum_node_3)
prod_node_2.add_child(sum_node_1)
prod_node_2.add_child(sum_node_4)
sum_layer_1 = SumLayer([sum_node_1, sum_node_2,
sum_node_3, sum_node_4])
# another product layer
prod_node_3 = ProductNode(var_scope=scope_1)
prod_node_4 = ProductNode(var_scope=scope_1)
prod_node_5 = ProductNode(var_scope=scope_4)
prod_node_6 = ProductNode(var_scope=scope_4)
sum_node_1.add_child(prod_node_3, 0.5)
sum_node_1.add_child(prod_node_4, 0.5)
sum_node_4.add_child(prod_node_5, 0.5)
sum_node_4.add_child(prod_node_6, 0.5)
prod_layer_2 = ProductLayer([prod_node_3, prod_node_4,
prod_node_5, prod_node_6])
# last sum one
scope_5 = frozenset({0})
scope_6 = frozenset({1})
sum_node_5 = SumNode(var_scope=scope_5)
sum_node_6 = SumNode(var_scope=scope_6)
sum_node_7 = SumNode(var_scope=scope_5)
sum_node_8 = SumNode(var_scope=scope_6)
sum_node_9 = SumNode(var_scope=scope_2)
sum_node_10 = SumNode(var_scope=scope_3)
sum_node_11 = SumNode(var_scope=scope_2)
sum_node_12 = SumNode(var_scope=scope_3)
prod_node_3.add_child(sum_node_5)
prod_node_3.add_child(sum_node_6)
prod_node_4.add_child(sum_node_7)
prod_node_4.add_child(sum_node_8)
prod_node_5.add_child(sum_node_9)
prod_node_5.add_child(sum_node_10)
prod_node_6.add_child(sum_node_11)
prod_node_6.add_child(sum_node_12)
sum_layer_2 = SumLayer([sum_node_5, sum_node_6,
sum_node_7, sum_node_8,
sum_node_9, sum_node_10,
sum_node_11, sum_node_12])
# input layer
vars = [2, 3, 2, 2]
input_layer = CategoricalIndicatorLayer(vars=vars)
last_sum_nodes = [sum_node_2, sum_node_3,
sum_node_5, sum_node_6,
sum_node_7, sum_node_8,
sum_node_9, sum_node_10,
sum_node_11, sum_node_12]
for sum_node in last_sum_nodes:
(var_scope,) = sum_node.var_scope
for input_node in input_layer.nodes():
if input_node.var == var_scope:
sum_node.add_child(input_node, 1.0)
spn = Spn(input_layer=input_layer,
layers=[sum_layer_2, prod_layer_2,
sum_layer_1, prod_layer_1,
root_layer])
# print(spn)
return spn
def linked_kernel_density_estimation(cls,
n_instances,
features,
node_dict=None,
alpha=0.1
# ,batch_size=1,
# sparse=False
):
"""
WRITEME
"""
n_features = len(features)
# the top one is a sum layer with a single node
root_node = SumNode()
root_layer = SumLayerLinked([root_node])
# second one is a product layer with n_instances nodes
product_nodes = [ProductNode() for i in range(n_instances)]
product_layer = ProductLayerLinked(product_nodes)
# linking them to the root node
for prod_node in product_nodes:
root_node.add_child(prod_node, 1. / n_instances)
# last layer can be a categorical smoothed input
# or sum_layer + categorical indicator input
input_layer = None
layers = None
n_leaf_nodes = n_features * n_instances
if node_dict is None:
# creating a sum_layer with n_leaf_nodes
sum_nodes = [SumNode() for i in range(n_leaf_nodes)]
# store them into a layer
sum_layer = SumLayerLinked(sum_nodes)
# linking them to the products above
for i, prod_node in enumerate(product_nodes):
for j in range(n_features):
# getting the next n_features nodes
prod_node.add_child(sum_nodes[i * n_features + j])
# now creating the indicator nodes
input_layer = \
CategoricalIndicatorLayerLinked(vars=features)
# linking the sum nodes to the indicator vars
for i, sum_node in enumerate(sum_nodes):
# getting the feature id
j = i % n_features
# and thus its number of values
n_values = features[j]
# getting the indices of indicators
start_index = sum(features[:j])
end_index = start_index + n_values
indicators = [node for node
in input_layer.nodes()][start_index:end_index]
for ind_node in indicators:
sum_node.add_child(ind_node, 1. / n_values)
# storing levels
layers = [sum_layer, product_layer,
root_layer]
else:
# create a categorical smoothed layer
input_layer = \
CategoricalSmoothedLayerLinked(vars=features,
node_dicts=node_dict,
alpha=alpha)
# it shall contain n_leaf_nodes nodes
smooth_nodes = list(input_layer.nodes())
assert len(smooth_nodes) == n_leaf_nodes
# linking it
for i, prod_node in enumerate(product_nodes):
for j in range(n_features):
# getting the next n_features nodes
prod_node.add_child(smooth_nodes[i * n_features + j])
# setting the used levels
layers = [product_layer, root_layer]
# create the spn from levels
kern_spn = SpnLinked(input_layer, layers)
return kern_spn
def test_spn_backprop():
# create initial layer
node1 = Node()
node2 = Node()
node3 = Node()
node4 = Node()
node5 = Node()
input_layer = CategoricalInputLayer([node1, node2,
node3, node4,
node5])
# top layer made by 3 sum nodes
sum1 = SumNode()
sum2 = SumNode()
sum3 = SumNode()
# linking to input nodes
weight11 = 0.3
sum1.add_child(node1, weight11)
weight12 = 0.3
sum1.add_child(node2, weight12)
weight13 = 0.4
sum1.add_child(node3, weight13)
weight22 = 0.15
sum2.add_child(node2, weight22)
weight23 = 0.15
sum2.add_child(node3, weight23)
weight24 = 0.7
sum2.add_child(node4, weight24)
weight33 = 0.4
sum3.add_child(node3, weight33)
weight34 = 0.25
sum3.add_child(node4, weight34)
weight35 = 0.35
sum3.add_child(node5, weight35)
sum_layer = SumLayer([sum1, sum2, sum3])
# another layer with two product nodes
prod1 = ProductNode()
prod2 = ProductNode()
prod1.add_child(sum1)
prod1.add_child(sum2)
prod2.add_child(sum2)
prod2.add_child(sum3)
prod_layer = ProductLayer([prod1, prod2])
# root layer, double sum
root1 = SumNode()
root2 = SumNode()
weightr11 = 0.5
root1.add_child(prod1, weightr11)
weightr12 = 0.5
root1.add_child(prod2, weightr12)
weightr21 = 0.9
root2.add_child(prod1, weightr21)
weightr22 = 0.1
root2.add_child(prod2, weightr22)
root_layer = SumLayer([root1, root2])
# root_layer = SumLayer([root1])
# create the spn
spn = Spn(input_layer=input_layer,
layers=[sum_layer, prod_layer, root_layer])
# setting the input values
val1 = 0.0
node1.set_val(val1)
val2 = 0.5
node2.set_val(val2)
val3 = 0.3
node3.set_val(val3)
val4 = 1.0
node4.set_val(val4)
val5 = 0.0
node5.set_val(val5)
# evaluating the spn
res = spn.test_eval()
print('spn eval\'d', res)
# backprop
spn.backprop()
# computing derivatives by hand
# topdown: root layer
root_der = 1.0
log_root_der = log(root_der)
# print('root ders', root1.log_der, root2.log_der)
print('root ders', root1.log_der)
assert_almost_equal(log_root_der, root1.log_der)
assert_almost_equal(log_root_der, root2.log_der)
# product layer
prod_der1 = (root_der * weightr11 +
root_der * weightr21)
prod_der2 = (root_der * weightr12 +
root_der * weightr22)
# prod_der1 = (root_der * weightr11)
# prod_der2 = (root_der * weightr12)
log_prod_der1 = log(prod_der1) if prod_der1 > 0.0 else LOG_ZERO
log_prod_der2 = log(prod_der2) if prod_der2 > 0.0 else LOG_ZERO
print('found prod ders', prod1.log_der, prod2.log_der)
print('expect prod ders', log_prod_der1, log_prod_der2)
if IS_LOG_ZERO(log_prod_der1):
assert IS_LOG_ZERO(prod1.log_der)
else:
assert_almost_equal(log_prod_der1, prod1.log_der)
if IS_LOG_ZERO(log_prod_der2):
assert IS_LOG_ZERO(prod2.log_der)
else:
assert_almost_equal(log_prod_der2, prod2.log_der)
# sum layer
sum_der1 = (
prod_der1 * (weight22 * val2 +
weight23 * val3 +
weight24 * val4))
log_sum_der1 = log(sum_der1) if sum_der1 > 0.0 else LOG_ZERO
sum_der2 = (prod_der1 * (weight11 * val1 +
weight12 * val2 +
weight13 * val3) +
prod_der2 * (weight33 * val3 +
weight34 * val4 +
weight35 * val5))
log_sum_der2 = log(sum_der2) if sum_der2 > 0.0 else LOG_ZERO
sum_der3 = (prod_der2 * (weight22 * val2 +
weight23 * val3 +
weight24 * val4))
log_sum_der3 = log(sum_der3) if sum_der3 > 0.0 else LOG_ZERO
print('expected sum ders', log_sum_der1,
log_sum_der2,
log_sum_der3)
print('found sum ders', sum1.log_der,
sum2.log_der,
sum3.log_der)
if IS_LOG_ZERO(log_sum_der1):
assert IS_LOG_ZERO(sum1.log_der)
else:
assert_almost_equal(log_sum_der1, sum1.log_der)
if IS_LOG_ZERO(log_sum_der2):
assert IS_LOG_ZERO(sum2.log_der)
else:
assert_almost_equal(log_sum_der2, sum2.log_der)
if IS_LOG_ZERO(log_sum_der3):
assert IS_LOG_ZERO(sum3.log_der)
else:
assert_almost_equal(log_sum_der3, sum3.log_der)
# final level, the first one
try:
log_der1 = log(sum_der1 * weight11)
except:
log_der1 = LOG_ZERO
try:
log_der2 = log(sum_der1 * weight12 +
sum_der2 * weight22)
except:
log_der2 = LOG_ZERO
try:
log_der3 = log(sum_der1 * weight13 +
sum_der2 * weight23 +
sum_der3 * weight33)
except:
log_der3 = LOG_ZERO
try:
log_der4 = log(sum_der2 * weight24 +
sum_der3 * weight34)
except:
log_der4 = LOG_ZERO
try:
log_der5 = log(sum_der3 * weight35)
except:
log_der5 = LOG_ZERO
# printing, just in case
print('child log der', node1.log_der, node2.log_der,
node3.log_der, node4.log_der, node5.log_der)
print('exact log der', log_der1, log_der2, log_der3,
log_der4, log_der5)
if IS_LOG_ZERO(log_der1):
assert IS_LOG_ZERO(node1.log_der)
else:
assert_almost_equal(log_der1, node1.log_der, 15)
if IS_LOG_ZERO(log_der2):
assert IS_LOG_ZERO(node2.log_der)
else:
assert_almost_equal(log_der2, node2.log_der, 15)
if IS_LOG_ZERO(log_der3):
assert IS_LOG_ZERO(node3.log_der)
else:
assert_almost_equal(log_der3, node3.log_der, 15)
if IS_LOG_ZERO(log_der4):
assert IS_LOG_ZERO(node4.log_der)
else:
assert_almost_equal(log_der4, node4.log_der, 15)
if IS_LOG_ZERO(log_der5):
assert IS_LOG_ZERO(node5.log_der)
else:
assert_almost_equal(log_der5, node5.log_der, 15)
def test_linked_to_theano_indicator():
# creating single nodes
root = SumNode()
prod1 = ProductNode()
prod2 = ProductNode()
prod3 = ProductNode()
sum1 = SumNode()
sum2 = SumNode()
sum3 = SumNode()
sum4 = SumNode()
ind1 = CategoricalIndicatorNode(var=0, var_val=0)
ind2 = CategoricalIndicatorNode(var=0, var_val=1)
ind3 = CategoricalIndicatorNode(var=1, var_val=0)
ind4 = CategoricalIndicatorNode(var=1, var_val=1)
ind5 = CategoricalIndicatorNode(var=2, var_val=0)
ind6 = CategoricalIndicatorNode(var=2, var_val=1)
ind7 = CategoricalIndicatorNode(var=2, var_val=2)
ind8 = CategoricalIndicatorNode(var=3, var_val=0)
ind9 = CategoricalIndicatorNode(var=3, var_val=1)
ind10 = CategoricalIndicatorNode(var=3, var_val=2)
ind11 = CategoricalIndicatorNode(var=3, var_val=3)
prod4 = ProductNode()
prod5 = ProductNode()
prod6 = ProductNode()
prod7 = ProductNode()
# linking nodes
root.add_child(prod1, 0.3)
root.add_child(prod2, 0.3)
root.add_child(prod3, 0.4)
prod1.add_child(sum1)
prod1.add_child(sum2)
prod2.add_child(ind7)
prod2.add_child(ind8)
prod2.add_child(ind11)
prod3.add_child(sum3)
prod3.add_child(sum4)
sum1.add_child(ind1, 0.3)
sum1.add_child(ind2, 0.3)
sum1.add_child(prod4, 0.4)
sum2.add_child(ind2, 0.5)
sum2.add_child(prod4, 0.2)
sum2.add_child(prod5, 0.3)
sum3.add_child(prod6, 0.5)
sum3.add_child(prod7, 0.5)
sum4.add_child(prod6, 0.5)
sum4.add_child(prod7, 0.5)
prod4.add_child(ind3)
prod4.add_child(ind4)
prod5.add_child(ind5)
prod5.add_child(ind6)
prod6.add_child(ind9)
prod6.add_child(ind10)
prod7.add_child(ind9)
prod7.add_child(ind10)
# building layers from nodes
root_layer = SumLayerLinked([root])
prod_layer = ProductLayerLinked([prod1, prod2, prod3])
sum_layer = SumLayerLinked([sum1, sum2, sum3, sum4])
aprod_layer = ProductLayerLinked([prod4, prod5, prod6, prod7])
ind_layer = CategoricalIndicatorLayer(nodes=[
ind1, ind2, ind3, ind4, ind5, ind6, ind7, ind8, ind9, ind10, ind11
])
# creating the linked spn
spn_linked = SpnLinked(
input_layer=ind_layer,
layers=[aprod_layer, sum_layer, prod_layer, root_layer])
print(spn_linked)
# converting to theano repr
spn_theano = SpnFactory.linked_to_theano(spn_linked)
print(spn_theano)
# time for some inference comparison
for instance in I:
print('linked')
res_l = spn_linked.eval(instance)
print(res_l)
print('theano')
res_t = spn_theano.eval(instance)
print(res_t)
assert_array_almost_equal(res_l, res_t)
def test_linked_to_theano_categorical():
vars = [2, 2, 3, 4]
freqs = [{
'var': 0,
'freqs': [1, 2]
}, {
'var': 1,
'freqs': [2, 2]
}, {
'var': 0,
'freqs': [3, 2]
}, {
'var': 1,
'freqs': [0, 3]
}, {
'var': 2,
'freqs': [1, 0, 2]
}, {
'var': 3,
'freqs': [1, 2, 1, 2]
}, {
'var': 3,
'freqs': [3, 4, 0, 1]
}]
# create input layer first
input_layer = CategoricalSmoothedLayer(vars=vars, node_dicts=freqs)
# get nodes
ind_nodes = [node for node in input_layer.nodes()]
root_node = ProductNode()
sum1 = SumNode()
sum2 = SumNode()
prod1 = ProductNode()
prod2 = ProductNode()
sum3 = SumNode()
sum4 = SumNode()
# linking
root_node.add_child(sum1)
root_node.add_child(sum2)
root_node.add_child(ind_nodes[0])
root_node.add_child(ind_nodes[1])
sum1.add_child(ind_nodes[2], 0.4)
sum1.add_child(ind_nodes[3], 0.6)
sum2.add_child(ind_nodes[3], 0.2)
sum2.add_child(prod1, 0.5)
sum2.add_child(prod2, 0.3)
prod1.add_child(ind_nodes[4])
prod1.add_child(sum3)
prod1.add_child(sum4)
prod2.add_child(sum3)
prod2.add_child(sum4)
sum3.add_child(ind_nodes[5], 0.5)
sum3.add_child(ind_nodes[6], 0.5)
sum4.add_child(ind_nodes[5], 0.4)
sum4.add_child(ind_nodes[6], 0.6)
# creating layers
root_layer = ProductLayerLinked([root_node])
sum_layer = SumLayerLinked([sum1, sum2])
prod_layer = ProductLayerLinked([prod1, prod2])
sum_layer2 = SumLayerLinked([sum3, sum4])
# create the linked spn
spn_linked = SpnLinked(
input_layer=input_layer,
layers=[sum_layer2, prod_layer, sum_layer, root_layer])
print(spn_linked)
# converting to theano repr
spn_theano = SpnFactory.linked_to_theano(spn_linked)
print(spn_theano)
# time for some inference comparison
for instance in I:
print('linked')
res_l = spn_linked.eval(instance)
print(res_l)
print('theano')
res_t = spn_theano.eval(instance)
print(res_t)
assert_array_almost_equal(res_l, res_t)
def test_layered_linked_spn():
# creating single nodes
# this code is replicated TODO: make a function
root = SumNode()
prod1 = ProductNode()
prod2 = ProductNode()
prod3 = ProductNode()
sum1 = SumNode()
sum2 = SumNode()
sum3 = SumNode()
sum4 = SumNode()
ind1 = CategoricalIndicatorNode(var=0, var_val=0)
ind2 = CategoricalIndicatorNode(var=0, var_val=1)
ind3 = CategoricalIndicatorNode(var=1, var_val=0)
ind4 = CategoricalIndicatorNode(var=1, var_val=1)
ind5 = CategoricalIndicatorNode(var=2, var_val=0)
ind6 = CategoricalIndicatorNode(var=2, var_val=1)
ind7 = CategoricalIndicatorNode(var=2, var_val=2)
ind8 = CategoricalIndicatorNode(var=3, var_val=0)
ind9 = CategoricalIndicatorNode(var=3, var_val=1)
ind10 = CategoricalIndicatorNode(var=3, var_val=2)
ind11 = CategoricalIndicatorNode(var=3, var_val=3)
prod4 = ProductNode()
prod5 = ProductNode()
prod6 = ProductNode()
prod7 = ProductNode()
# linking nodes
root.add_child(prod1, 0.3)
root.add_child(prod2, 0.3)
root.add_child(prod3, 0.4)
prod1.add_child(sum1)
prod1.add_child(sum2)
prod2.add_child(ind7)
prod2.add_child(ind8)
prod2.add_child(ind11)
prod3.add_child(sum3)
prod3.add_child(sum4)
sum1.add_child(ind1, 0.3)
sum1.add_child(ind2, 0.3)
sum1.add_child(prod4, 0.4)
sum2.add_child(ind2, 0.5)
sum2.add_child(prod4, 0.2)
sum2.add_child(prod5, 0.3)
sum3.add_child(prod6, 0.5)
sum3.add_child(prod7, 0.5)
sum4.add_child(prod6, 0.5)
sum4.add_child(prod7, 0.5)
prod4.add_child(ind3)
prod4.add_child(ind4)
prod5.add_child(ind5)
prod5.add_child(ind6)
prod6.add_child(ind9)
prod6.add_child(ind10)
prod7.add_child(ind9)
prod7.add_child(ind10)
spn = SpnFactory.layered_linked_spn(root)
print(spn)
print(spn.stats())
def test_mini_spn_fit_em():
vars = numpy.array([2, 2, 2, 2])
input_layer = CategoricalIndicatorLayer(vars=vars)
print(input_layer)
ind1 = input_layer._nodes[0]
ind2 = input_layer._nodes[1]
ind3 = input_layer._nodes[2]
ind4 = input_layer._nodes[3]
ind5 = input_layer._nodes[4]
ind6 = input_layer._nodes[5]
ind7 = input_layer._nodes[6]
ind8 = input_layer._nodes[7]
# creating a sum layer of 4 nodes
sum1 = SumNode()
sum2 = SumNode()
sum3 = SumNode()
sum4 = SumNode()
sum1.add_child(ind1, 0.6)
sum1.add_child(ind2, 0.4)
sum2.add_child(ind3, 0.5)
sum2.add_child(ind4, 0.5)
sum3.add_child(ind5, 0.7)
sum3.add_child(ind6, 0.3)
sum4.add_child(ind7, 0.4)
sum4.add_child(ind8, 0.6)
sum_layer = SumLayer(nodes=[sum1, sum2, sum3, sum4])
# and a top layer of 3 products
prod1 = ProductNode()
prod2 = ProductNode()
prod3 = ProductNode()
prod1.add_child(sum1)
prod1.add_child(sum2)
prod2.add_child(sum2)
prod2.add_child(sum3)
prod3.add_child(sum3)
prod3.add_child(sum4)
prod_layer = ProductLayer(nodes=[prod1, prod2, prod3])
# root layer
root = SumNode()
root.add_child(prod1, 0.4)
root.add_child(prod2, 0.25)
root.add_child(prod3, 0.35)
root_layer = SumLayer(nodes=[root])
spn = Spn(input_layer=input_layer,
layers=[sum_layer, prod_layer, root_layer])
print(spn)
# training on obs
spn.fit_em(train=syn_train_data, valid=syn_val_data, test=None, hard=True)
def test_sum_layer_backprop():
# input layer made of 5 generic nodes
node1 = Node()
node2 = Node()
node3 = Node()
node4 = Node()
node5 = Node()
# top layer made by 3 sum nodes
sum1 = SumNode()
sum2 = SumNode()
sum3 = SumNode()
# linking to input nodes
weight11 = 0.3
sum1.add_child(node1, weight11)
weight12 = 0.3
sum1.add_child(node2, weight12)
weight13 = 0.4
sum1.add_child(node3, weight13)
weight22 = 0.15
sum2.add_child(node2, weight22)
weight23 = 0.15
sum2.add_child(node3, weight23)
weight24 = 0.7
sum2.add_child(node4, weight24)
weight33 = 0.4
sum3.add_child(node3, weight33)
weight34 = 0.25
sum3.add_child(node4, weight34)
weight35 = 0.35
sum3.add_child(node5, weight35)
sum_layer = SumLayer([sum1, sum2, sum3])
# setting input values
val1 = 0.0
node1.set_val(val1)
val2 = 0.5
node2.set_val(val2)
val3 = 0.3
node3.set_val(val3)
val4 = 1.0
node4.set_val(val4)
val5 = 0.0
node5.set_val(val5)
# evaluating
sum_layer.eval()
print('eval\'d layer:', sum_layer.node_values())
# set the parent derivatives
sum_der1 = 1.0
sum1.log_der = log(sum_der1)
sum_der2 = 1.0
sum2.log_der = log(sum_der2)
sum_der3 = 0.0
sum3.log_der = LOG_ZERO
# back prop layer wise
sum_layer.backprop()
# check for correctness
try:
log_der1 = log(sum_der1 * weight11)
except:
log_der1 = LOG_ZERO
try:
log_der2 = log(sum_der1 * weight12 +
sum_der2 * weight22)
except:
log_der2 = LOG_ZERO
try:
log_der3 = log(sum_der1 * weight13 +
sum_der2 * weight23 +
sum_der3 * weight33)
except:
log_der3 = LOG_ZERO
try:
log_der4 = log(sum_der2 * weight24 +
sum_der3 * weight34)
except:
log_der4 = LOG_ZERO
try:
log_der5 = log(sum_der3 * weight35)
except:
log_der5 = LOG_ZERO
# printing, just in case
print('child log der', node1.log_der, node2.log_der,
node3.log_der, node4.log_der, node5.log_der)
print('exact log der', log_der1, log_der2, log_der3,
log_der4, log_der5)
if IS_LOG_ZERO(log_der1):
assert IS_LOG_ZERO(node1.log_der)
else:
assert_almost_equal(log_der1, node1.log_der, 15)
if IS_LOG_ZERO(log_der2):
assert IS_LOG_ZERO(node2.log_der)
else:
assert_almost_equal(log_der2, node2.log_der, 15)
if IS_LOG_ZERO(log_der3):
assert IS_LOG_ZERO(node3.log_der)
else:
assert_almost_equal(log_der3, node3.log_der, 15)
if IS_LOG_ZERO(log_der4):
assert IS_LOG_ZERO(node4.log_der)
else:
assert_almost_equal(log_der4, node4.log_der, 15)
if IS_LOG_ZERO(log_der5):
assert IS_LOG_ZERO(node5.log_der)
else:
assert_almost_equal(log_der5, node5.log_der, 15)
# updating weights
eta = 0.1
sum_layer.update_weights(Spn.test_weight_update, 0)
# checking for correctness
weight_u11 = sum_der1 * val1 * eta + weight11
weight_u12 = sum_der1 * val2 * eta + weight12
weight_u13 = sum_der1 * val3 * eta + weight13
weight_u22 = sum_der2 * val2 * eta + weight22
weight_u23 = sum_der2 * val3 * eta + weight23
weight_u24 = sum_der2 * val4 * eta + weight24
weight_u33 = sum_der3 * val3 * eta + weight33
weight_u34 = sum_der3 * val4 * eta + weight34
weight_u35 = sum_der3 * val5 * eta + weight35
# normalizing
weight_sum1 = weight_u11 + weight_u12 + weight_u13
weight_sum2 = weight_u22 + weight_u23 + weight_u24
weight_sum3 = weight_u33 + weight_u34 + weight_u35
weight_u11 = weight_u11 / weight_sum1
weight_u12 = weight_u12 / weight_sum1
weight_u13 = weight_u13 / weight_sum1
weight_u22 = weight_u22 / weight_sum2
weight_u23 = weight_u23 / weight_sum2
weight_u24 = weight_u24 / weight_sum2
weight_u33 = weight_u33 / weight_sum3
weight_u34 = weight_u34 / weight_sum3
weight_u35 = weight_u35 / weight_sum3
print('expected weights', weight_u11, weight_u12, weight_u13,
weight_u22, weight_u23, weight_u24,
weight_u33, weight_u34, weight_u35)
print('found weights', sum1.weights[0], sum1.weights[1], sum1.weights[2],
sum2.weights[0], sum2.weights[1], sum2.weights[2],
sum3.weights[0], sum3.weights[1], sum3.weights[2])
assert_almost_equal(weight_u11, sum1.weights[0], 10)
assert_almost_equal(weight_u12, sum1.weights[1], 10)
assert_almost_equal(weight_u13, sum1.weights[2], 10)
assert_almost_equal(weight_u22, sum2.weights[0], 10)
assert_almost_equal(weight_u23, sum2.weights[1], 10)
assert_almost_equal(weight_u24, sum2.weights[2], 10)
assert_almost_equal(weight_u33, sum3.weights[0], 10)
assert_almost_equal(weight_u34, sum3.weights[1], 10)
assert_almost_equal(weight_u35, sum3.weights[2], 10)
#
# resetting derivatives
#
node1.log_der = LOG_ZERO
node2.log_der = LOG_ZERO
node3.log_der = LOG_ZERO
node4.log_der = LOG_ZERO
node5.log_der = LOG_ZERO
# setting new values as inputs
val1 = 0.0
node1.set_val(val1)
val2 = 0.0
node2.set_val(val2)
val3 = 0.3
node3.set_val(val3)
val4 = 1.0
node4.set_val(val4)
val5 = 1.0
node5.set_val(val5)
# evaluating again
sum_layer.eval()
print('eval\'d layer:', sum_layer.node_values())
# set the parent derivatives
sum_der1 = 1.0
sum1.log_der = log(sum_der1)
sum_der2 = 1.0
sum2.log_der = log(sum_der2)
sum_der3 = 0.0
sum3.log_der = LOG_ZERO
# back prop layer wise
sum_layer.backprop()
# check for correctness
try:
log_der1 = log(sum_der1 * weight_u11)
except:
log_der1 = LOG_ZERO
try:
log_der2 = log(sum_der1 * weight_u12 +
sum_der2 * weight_u22)
except:
log_der2 = LOG_ZERO
try:
log_der3 = log(sum_der1 * weight_u13 +
sum_der2 * weight_u23 +
sum_der3 * weight_u33)
except:
log_der3 = LOG_ZERO
try:
log_der4 = log(sum_der2 * weight_u24 +
sum_der3 * weight_u34)
except:
log_der4 = LOG_ZERO
try:
log_der5 = log(sum_der3 * weight_u35)
except:
log_der5 = LOG_ZERO
# printing, just in case
print('child log der', node1.log_der, node2.log_der,
node3.log_der, node4.log_der, node5.log_der)
print('exact log der', log_der1, log_der2, log_der3,
log_der4, log_der5)
if IS_LOG_ZERO(log_der1):
assert IS_LOG_ZERO(node1.log_der)
else:
assert_almost_equal(log_der1, node1.log_der, 15)
if IS_LOG_ZERO(log_der2):
assert IS_LOG_ZERO(node2.log_der)
else:
assert_almost_equal(log_der2, node2.log_der, 15)
if IS_LOG_ZERO(log_der3):
assert IS_LOG_ZERO(node3.log_der)
else:
assert_almost_equal(log_der3, node3.log_der, 15)
if IS_LOG_ZERO(log_der4):
assert IS_LOG_ZERO(node4.log_der)
else:
assert_almost_equal(log_der4, node4.log_der, 15)
if IS_LOG_ZERO(log_der5):
assert IS_LOG_ZERO(node5.log_der)
else:
assert_almost_equal(log_der5, node5.log_der, 15)
def linked_categorical_input_to_indicators(spn, input_layer=None):
"""
Convertes a linked spn categorical input layer into an indicator one
"""
#
# get child, parent relations for node relinking
child_assoc = retrieve_children_parent_assoc(spn)
#
# get input layer
cat_input_layer = spn.input_layer()
assert isinstance(cat_input_layer, CategoricalSmoothedLayerLinked)
#
# one indicator node for each var value
vars = cat_input_layer.vars()
if not vars:
vars = list(sorted({node.var for node in cat_input_layer.nodes()}))
feature_values = cat_input_layer.feature_vals()
# print('vars', vars)
# print('feature values', feature_values)
indicator_nodes = [
CategoricalIndicatorNode(var, val) for i, var in enumerate(vars)
for val in range(feature_values[i])
]
# for node in indicator_nodes:
# print(node)
indicator_map = defaultdict(set)
for ind_node in indicator_nodes:
indicator_map[ind_node.var].add(ind_node)
sum_nodes = []
#
# as many sum nodes as cat nodes
for node in cat_input_layer.nodes():
sum_node = SumNode(var_scope=frozenset([node.var]))
sum_nodes.append(sum_node)
for ind_node in sorted(indicator_map[node.var],
key=lambda x: x.var_val):
sum_node.add_child(ind_node,
numpy.exp(node._var_probs[ind_node.var_val]))
#
# removing links to parents
parents = child_assoc[node]
for p_node in parents:
#
# assume it to be a product node
# TODO: generalize
assert isinstance(p_node, ProductNode)
p_node.children.remove(node)
p_node.add_child(sum_node)
#
# creating layer
sum_layer = SumLayerLinked(sum_nodes)
indicator_layer = CategoricalIndicatorLayerLinked(indicator_nodes)
cat_input_layer.disconnect_layer()
spn.set_input_layer(indicator_layer)
spn.insert_layer(sum_layer, 0)
return spn
def test_spn_set_get_weights():
# create a simple spn
root_node = SumNode()
root_layer = SumLayer([root_node])
prod_node_1 = ProductNode()
prod_node_2 = ProductNode()
root_node.add_child(prod_node_1, 0.5)
root_node.add_child(prod_node_2, 0.5)
prod_layer = ProductLayer([prod_node_1,
prod_node_2])
sum_node_1 = SumNode()
sum_node_2 = SumNode()
sum_node_3 = SumNode()
prod_node_1.add_child(sum_node_1)
prod_node_1.add_child(sum_node_2)
prod_node_2.add_child(sum_node_2)
prod_node_2.add_child(sum_node_3)
sum_layer = SumLayer([sum_node_1, sum_node_2,
sum_node_3])
ind_node_1 = CategoricalIndicatorNode(var=0, var_val=1)
ind_node_2 = CategoricalIndicatorNode(var=0, var_val=1)
ind_node_3 = CategoricalIndicatorNode(var=0, var_val=1)
ind_node_4 = CategoricalIndicatorNode(var=0, var_val=1)
ind_node_5 = CategoricalIndicatorNode(var=0, var_val=1)
input_layer = CategoricalInputLayer(nodes=[ind_node_1,
ind_node_2,
ind_node_3,
ind_node_4,
ind_node_5])
sum_node_1.add_child(ind_node_1, 0.2)
sum_node_1.add_child(ind_node_2, 0.2)
sum_node_2.add_child(ind_node_2, 0.2)
sum_node_2.add_child(ind_node_3, 0.2)
sum_node_2.add_child(ind_node_4, 0.2)
sum_node_3.add_child(ind_node_4, 0.2)
sum_node_3.add_child(ind_node_5, 0.2)
spn = Spn(input_layer=input_layer,
layers=[sum_layer, prod_layer, root_layer])
print(spn)
# storing these weights
curr_weights = spn.get_weights()
# setting the new weights
spn.set_weights(weights_ds)
# getting them again
new_weights = spn.get_weights()
# comparing them
assert new_weights == weights_ds
# now setting back the previous one
spn.set_weights(curr_weights)
# getting them back again
old_weights = spn.get_weights()
# and checking
assert old_weights == curr_weights
def test_linked_to_theano_categorical():
vars = [2, 2, 3, 4]
freqs = [{'var': 0, 'freqs': [1, 2]},
{'var': 1, 'freqs': [2, 2]},
{'var': 0, 'freqs': [3, 2]},
{'var': 1, 'freqs': [0, 3]},
{'var': 2, 'freqs': [1, 0, 2]},
{'var': 3, 'freqs': [1, 2, 1, 2]},
{'var': 3, 'freqs': [3, 4, 0, 1]}]
# create input layer first
input_layer = CategoricalSmoothedLayer(vars=vars,
node_dicts=freqs)
# get nodes
ind_nodes = [node for node in input_layer.nodes()]
root_node = ProductNode()
sum1 = SumNode()
sum2 = SumNode()
prod1 = ProductNode()
prod2 = ProductNode()
sum3 = SumNode()
sum4 = SumNode()
# linking
root_node.add_child(sum1)
root_node.add_child(sum2)
root_node.add_child(ind_nodes[0])
root_node.add_child(ind_nodes[1])
sum1.add_child(ind_nodes[2], 0.4)
sum1.add_child(ind_nodes[3], 0.6)
sum2.add_child(ind_nodes[3], 0.2)
sum2.add_child(prod1, 0.5)
sum2.add_child(prod2, 0.3)
prod1.add_child(ind_nodes[4])
prod1.add_child(sum3)
prod1.add_child(sum4)
prod2.add_child(sum3)
prod2.add_child(sum4)
sum3.add_child(ind_nodes[5], 0.5)
sum3.add_child(ind_nodes[6], 0.5)
sum4.add_child(ind_nodes[5], 0.4)
sum4.add_child(ind_nodes[6], 0.6)
# creating layers
root_layer = ProductLayerLinked([root_node])
sum_layer = SumLayerLinked([sum1, sum2])
prod_layer = ProductLayerLinked([prod1, prod2])
sum_layer2 = SumLayerLinked([sum3, sum4])
# create the linked spn
spn_linked = SpnLinked(input_layer=input_layer,
layers=[sum_layer2, prod_layer,
sum_layer, root_layer])
print(spn_linked)
# converting to theano repr
spn_theano = SpnFactory.linked_to_theano(spn_linked)
print(spn_theano)
# time for some inference comparison
for instance in I:
print('linked')
res_l = spn_linked.eval(instance)
print(res_l)
print('theano')
res_t = spn_theano.eval(instance)
print(res_t)
assert_array_almost_equal(res_l, res_t)
def build_linked_layered_spn(print_spn=True):
#
# building an indicator layer
ind_x_00 = CategoricalIndicatorNode(0, 0)
ind_x_01 = CategoricalIndicatorNode(0, 1)
ind_x_10 = CategoricalIndicatorNode(1, 0)
ind_x_11 = CategoricalIndicatorNode(1, 1)
ind_x_20 = CategoricalIndicatorNode(2, 0)
ind_x_21 = CategoricalIndicatorNode(2, 1)
input_layer = CategoricalIndicatorLayer(
[ind_x_00, ind_x_01, ind_x_10, ind_x_11, ind_x_20, ind_x_21])
#
# sum layer
#
sum_node_1 = SumNode(frozenset([0]))
sum_node_1.add_child(ind_x_00, 0.1)
sum_node_1.add_child(ind_x_01, 0.9)
sum_node_2 = SumNode(frozenset([0]))
sum_node_2.add_child(ind_x_00, 0.4)
sum_node_2.add_child(ind_x_01, 0.6)
sum_node_3 = SumNode(frozenset([1]))
sum_node_3.add_child(ind_x_10, 0.3)
sum_node_3.add_child(ind_x_11, 0.7)
sum_node_4 = SumNode(frozenset([1]))
sum_node_4.add_child(ind_x_10, 0.6)
sum_node_4.add_child(ind_x_11, 0.4)
sum_node_5 = SumNode(frozenset([2]))
sum_node_5.add_child(ind_x_20, 0.5)
sum_node_5.add_child(ind_x_21, 0.5)
sum_node_6 = SumNode(frozenset([2]))
sum_node_6.add_child(ind_x_20, 0.2)
sum_node_6.add_child(ind_x_21, 0.8)
sum_layer_1 = SumLayerLinked([
sum_node_1, sum_node_2, sum_node_3, sum_node_4, sum_node_5, sum_node_6
])
#
# product nodes
#
# xy
prod_node_7 = ProductNode(frozenset([0, 1]))
prod_node_7.add_child(sum_node_1)
prod_node_7.add_child(sum_node_3)
prod_node_8 = ProductNode(frozenset([0, 1]))
prod_node_8.add_child(sum_node_2)
prod_node_8.add_child(sum_node_4)
prod_node_9 = ProductNode(frozenset([0, 1]))
prod_node_9.add_child(sum_node_1)
prod_node_9.add_child(sum_node_3)
#
# yz
prod_node_10 = ProductNode(frozenset([1, 2]))
prod_node_10.add_child(sum_node_4)
prod_node_10.add_child(sum_node_5)
prod_node_11 = ProductNode(frozenset([1, 2]))
prod_node_11.add_child(sum_node_4)
prod_node_11.add_child(sum_node_6)
prod_layer_2 = ProductLayerLinked(
[prod_node_7, prod_node_8, prod_node_9, prod_node_10, prod_node_11])
#
# sum nodes
#
# xy
sum_node_12 = SumNode(frozenset([0, 1]))
sum_node_12.add_child(prod_node_7, 0.1)
sum_node_12.add_child(prod_node_8, 0.9)
sum_node_13 = SumNode(frozenset([0, 1]))
sum_node_13.add_child(prod_node_8, 0.7)
sum_node_13.add_child(prod_node_9, 0.3)
#
# yz
sum_node_14 = SumNode(frozenset([1, 2]))
sum_node_14.add_child(prod_node_10, 0.6)
sum_node_14.add_child(prod_node_11, 0.4)
sum_layer_3 = SumLayerLinked([sum_node_12, sum_node_13, sum_node_14])
#
# product nodes
prod_node_15 = ProductNode(frozenset([0, 1, 2]))
prod_node_15.add_child(sum_node_12)
prod_node_15.add_child(sum_node_6)
prod_node_16 = ProductNode(frozenset([0, 1, 2]))
prod_node_16.add_child(sum_node_13)
prod_node_16.add_child(sum_node_5)
prod_node_17 = ProductNode(frozenset([0, 1, 2]))
prod_node_17.add_child(sum_node_2)
prod_node_17.add_child(sum_node_14)
prod_layer_4 = ProductLayerLinked(
[prod_node_15, prod_node_16, prod_node_17])
#
# root
sum_node_18 = SumNode(frozenset([0, 1, 2]))
sum_node_18.add_child(prod_node_15, 0.2)
sum_node_18.add_child(prod_node_16, 0.2)
sum_node_18.add_child(prod_node_17, 0.6)
sum_layer_5 = SumLayerLinked([sum_node_18])
#
# creating the spn
layers = [
sum_layer_1, prod_layer_2, sum_layer_3, prod_layer_4, sum_layer_5
]
nodes = [node for layer in layers for node in layer.nodes()]
spn = SpnLinked(input_layer=input_layer, layers=layers)
if print_spn:
print(spn)
return spn, layers, nodes
def test_categorical_to_indicator_input_layer():
#
# creating all the data slices
# the slicing is a fake stub
# rows = 5
# cols = 5
var_1 = 0
values_1 = 2
var_2 = 1
values_2 = 3
var_3 = 2
values_3 = 4
node_1 = SumNode()
node_1.id = 1
node_2 = ProductNode()
node_2.id = 2
node_3 = SumNode()
node_3.id = 3
# adding first level
weight_12 = 0.4
weight_13 = 0.6
node_1.add_child(node_2, weight_12)
node_1.add_child(node_3, weight_13)
node_4 = ProductNode()
node_4.id = 4
leaf_5 = CategoricalSmoothedNode(var_1, values_1)
leaf_5.id = 5
# not adding the slice to the stack
node_2.add_child(node_4)
node_2.add_child(leaf_5)
node_6 = SumNode()
node_6.id = 6
node_7 = SumNode()
node_7.id = 7
weight_36 = 0.1
weight_37 = 0.9
node_3.add_child(node_6, weight_36)
node_3.add_child(node_7, weight_37)
node_8 = ProductNode()
node_8.id = 8
leaf_15 = CategoricalSmoothedNode(var_2, values_2)
leaf_15.id = 15
node_4.add_child(node_8)
node_4.add_child(leaf_15)
leaf_13 = CategoricalSmoothedNode(var_3, values_3)
leaf_13.id = 13
leaf_14 = CategoricalSmoothedNode(var_1, values_1)
leaf_14.id = 14
node_8.add_child(leaf_13)
node_8.add_child(leaf_14)
node_9 = ProductNode()
node_9.id = 9
leaf_16 = CategoricalSmoothedNode(var_2, values_2)
leaf_16.id = 16
leaf_17 = CategoricalSmoothedNode(var_3, values_3)
leaf_17.id = 17
node_9.add_child(leaf_16)
node_9.add_child(leaf_17)
node_10 = ProductNode()
node_10.id = 10
leaf_18 = CategoricalSmoothedNode(var_2, values_2)
leaf_18.id = 18
leaf_19 = CategoricalSmoothedNode(var_2, values_2)
leaf_19.id = 19
node_10.add_child(leaf_18)
node_10.add_child(leaf_19)
weight_69 = 0.3
weight_610 = 0.7
node_6.add_child(node_9, weight_69)
node_6.add_child(node_10, weight_610)
node_11 = ProductNode()
node_11.id = 11
leaf_20 = CategoricalSmoothedNode(var_1, values_1)
leaf_20.id = 20
leaf_21 = CategoricalSmoothedNode(var_3, values_3)
leaf_21.id = 21
node_11.add_child(leaf_20)
node_11.add_child(leaf_21)
node_12 = ProductNode()
node_12.id = 12
leaf_22 = CategoricalSmoothedNode(var_1, values_1)
leaf_22.id = 22
leaf_23 = CategoricalSmoothedNode(var_3, values_3)
leaf_23.id = 23
node_12.add_child(leaf_22)
node_12.add_child(leaf_23)
weight_711 = 0.5
weight_712 = 0.5
node_7.add_child(node_11, weight_711)
node_7.add_child(node_12, weight_712)
root_node = SpnFactory.layered_pruned_linked_spn(node_1)
print('ROOT nODE', root_node)
spn = SpnFactory.layered_linked_spn(root_node)
print('SPN', spn)
assert spn.n_layers() == 3
for i, layer in enumerate(spn.top_down_layers()):
if i == 0:
assert layer.n_nodes() == 1
elif i == 1:
assert layer.n_nodes() == 5
elif i == 2:
assert layer.n_nodes() == 12
#
# changing input layer
spn = linked_categorical_input_to_indicators(spn)
print('Changed input layer to indicator variables')
print(spn)
def test_layered_pruned_linked_spn_cltree():
#
# creating all the data slices
# the slicing is a fake stub
rows = 5
cols = 5
var = 1
values = 2
vars = [2, 3]
var_values = [2, 2]
s_data = numpy.array([[0, 1], [1, 1], [1, 0], [0, 0]])
node_1 = SumNode()
node_1.id = 1
node_2 = ProductNode()
node_2.id = 2
node_3 = SumNode()
node_3.id = 3
# adding first level
weight_12 = 0.4
weight_13 = 0.6
node_1.add_child(node_2, weight_12)
node_1.add_child(node_3, weight_13)
node_4 = ProductNode()
node_4.id = 4
leaf_5 = CategoricalSmoothedNode(var, values)
leaf_5.id = 5
# not adding the slice to the stack
node_2.add_child(node_4)
node_2.add_child(leaf_5)
node_6 = SumNode()
node_6.id = 6
node_7 = SumNode()
node_7.id = 7
weight_36 = 0.1
weight_37 = 0.9
node_3.add_child(node_6, weight_36)
node_3.add_child(node_7, weight_37)
node_8 = ProductNode()
node_8.id = 8
#
# this is a cltree
leaf_15 = CLTreeNode(vars=vars, var_values=var_values, data=s_data)
leaf_15.id = 15
node_4.add_child(node_8)
node_4.add_child(leaf_15)
leaf_13 = CategoricalSmoothedNode(var, values)
leaf_13.id = 13
leaf_14 = CLTreeNode(vars=vars, var_values=var_values, data=s_data)
leaf_14.id = 14
node_8.add_child(leaf_13)
node_8.add_child(leaf_14)
leaf_9 = CLTreeNode(vars=vars, var_values=var_values, data=s_data)
leaf_9.id = 9
node_10 = ProductNode()
node_10.id = 10
leaf_18 = CategoricalSmoothedNode(var, values)
leaf_18.id = 18
leaf_19 = CategoricalSmoothedNode(var, values)
leaf_19.id = 19
node_10.add_child(leaf_18)
node_10.add_child(leaf_19)
weight_69 = 0.3
weight_610 = 0.7
node_6.add_child(leaf_9, weight_69)
node_6.add_child(node_10, weight_610)
node_11 = ProductNode()
node_11.id = 11
leaf_20 = CategoricalSmoothedNode(var, values)
leaf_20.id = 20
leaf_21 = CategoricalSmoothedNode(var, values)
leaf_21.id = 21
node_11.add_child(leaf_20)
node_11.add_child(leaf_21)
node_12 = ProductNode()
node_12.id = 12
leaf_22 = CLTreeNode(vars=vars, var_values=var_values, data=s_data)
leaf_22.id = 22
leaf_23 = CategoricalSmoothedNode(var, values)
leaf_23.id = 23
node_12.add_child(leaf_22)
node_12.add_child(leaf_23)
weight_711 = 0.5
weight_712 = 0.5
node_7.add_child(node_11, weight_711)
node_7.add_child(node_12, weight_712)
print('Added nodes')
root_node = SpnFactory.layered_pruned_linked_spn(node_1)
print('ROOT nODE', root_node)
spn = SpnFactory.layered_linked_spn(root_node)
print('SPN', spn)
assert spn.n_layers() == 3
for i, layer in enumerate(spn.top_down_layers()):
if i == 0:
assert layer.n_nodes() == 1
elif i == 1:
assert layer.n_nodes() == 4
elif i == 2:
assert layer.n_nodes() == 10
def test_sum_layer_create_and_eval():
# creating generic nodes
node1 = Node()
node2 = Node()
node3 = Node()
# whose values are
val1 = 1.
val2 = 1.
val3 = 0.
node1.set_val(val1)
node2.set_val(val2)
node3.set_val(val3)
# setting weights
weight11 = 0.2
weight12 = 0.3
weight13 = 0.5
weight21 = 0.3
weight22 = 0.7
weight32 = 0.4
weight33 = 0.6
# creating sum nodes
sum1 = SumNode()
sum2 = SumNode()
sum3 = SumNode()
# adding children
sum1.add_child(node1, weight11)
sum1.add_child(node2, weight12)
sum1.add_child(node3, weight13)
sum2.add_child(node1, weight21)
sum2.add_child(node2, weight22)
sum3.add_child(node2, weight32)
sum3.add_child(node3, weight33)
# adding to layer
sum_layer = SumLayer([sum1, sum2, sum3])
# evaluation
sum_layer.eval()
# computing 'log values by hand'
layer_evals = sum_layer.node_values()
print('Layer eval nodes')
print(layer_evals)
logval1 = log(weight11 * val1 +
weight12 * val2 +
weight13 * val3)
logval2 = log(weight21 * val1 +
weight22 * val2)
logval3 = log(weight32 * val2 +
weight33 * val3)
logvals = [logval1, logval2, logval3]
print('log vals')
print(logvals)
# checking for correctness
for logval, eval in zip(logvals, layer_evals):
assert_almost_equal(logval, eval, PRECISION)
def test_layered_linked_spn():
# creating single nodes
# this code is replicated TODO: make a function
root = SumNode()
prod1 = ProductNode()
prod2 = ProductNode()
prod3 = ProductNode()
sum1 = SumNode()
sum2 = SumNode()
sum3 = SumNode()
sum4 = SumNode()
ind1 = CategoricalIndicatorNode(var=0, var_val=0)
ind2 = CategoricalIndicatorNode(var=0, var_val=1)
ind3 = CategoricalIndicatorNode(var=1, var_val=0)
ind4 = CategoricalIndicatorNode(var=1, var_val=1)
ind5 = CategoricalIndicatorNode(var=2, var_val=0)
ind6 = CategoricalIndicatorNode(var=2, var_val=1)
ind7 = CategoricalIndicatorNode(var=2, var_val=2)
ind8 = CategoricalIndicatorNode(var=3, var_val=0)
ind9 = CategoricalIndicatorNode(var=3, var_val=1)
ind10 = CategoricalIndicatorNode(var=3, var_val=2)
ind11 = CategoricalIndicatorNode(var=3, var_val=3)
prod4 = ProductNode()
prod5 = ProductNode()
prod6 = ProductNode()
prod7 = ProductNode()
# linking nodes
root.add_child(prod1, 0.3)
root. add_child(prod2, 0.3)
root.add_child(prod3, 0.4)
prod1.add_child(sum1)
prod1.add_child(sum2)
prod2.add_child(ind7)
prod2.add_child(ind8)
prod2.add_child(ind11)
prod3.add_child(sum3)
prod3.add_child(sum4)
sum1.add_child(ind1, 0.3)
sum1.add_child(ind2, 0.3)
sum1.add_child(prod4, 0.4)
sum2.add_child(ind2, 0.5)
sum2.add_child(prod4, 0.2)
sum2.add_child(prod5, 0.3)
sum3.add_child(prod6, 0.5)
sum3.add_child(prod7, 0.5)
sum4.add_child(prod6, 0.5)
sum4.add_child(prod7, 0.5)
prod4.add_child(ind3)
prod4.add_child(ind4)
prod5.add_child(ind5)
prod5.add_child(ind6)
prod6.add_child(ind9)
prod6.add_child(ind10)
prod7.add_child(ind9)
prod7.add_child(ind10)
spn = SpnFactory.layered_linked_spn(root)
print(spn)
print(spn.stats())
def test_layered_pruned_linked_spn_cltree():
#
# creating all the data slices
# the slicing is a fake stub
rows = 5
cols = 5
var = 1
values = 2
vars = [2, 3]
var_values = [2, 2]
s_data = numpy.array([[0, 1], [1, 1], [1, 0], [0, 0]])
node_1 = SumNode()
node_1.id = 1
node_2 = ProductNode()
node_2.id = 2
node_3 = SumNode()
node_3.id = 3
# adding first level
weight_12 = 0.4
weight_13 = 0.6
node_1.add_child(node_2, weight_12)
node_1.add_child(node_3, weight_13)
node_4 = ProductNode()
node_4.id = 4
leaf_5 = CategoricalSmoothedNode(var,
values)
leaf_5.id = 5
# not adding the slice to the stack
node_2.add_child(node_4)
node_2.add_child(leaf_5)
node_6 = SumNode()
node_6.id = 6
node_7 = SumNode()
node_7.id = 7
weight_36 = 0.1
weight_37 = 0.9
node_3.add_child(node_6, weight_36)
node_3.add_child(node_7, weight_37)
node_8 = ProductNode()
node_8.id = 8
#
# this is a cltree
leaf_15 = CLTreeNode(vars=vars,
var_values=var_values,
data=s_data)
leaf_15.id = 15
node_4.add_child(node_8)
node_4.add_child(leaf_15)
leaf_13 = CategoricalSmoothedNode(var,
values)
leaf_13.id = 13
leaf_14 = CLTreeNode(vars=vars,
var_values=var_values,
data=s_data)
leaf_14.id = 14
node_8.add_child(leaf_13)
node_8.add_child(leaf_14)
leaf_9 = CLTreeNode(vars=vars,
var_values=var_values,
data=s_data)
leaf_9.id = 9
node_10 = ProductNode()
node_10.id = 10
leaf_18 = CategoricalSmoothedNode(var,
values)
leaf_18.id = 18
leaf_19 = CategoricalSmoothedNode(var,
values)
leaf_19.id = 19
node_10.add_child(leaf_18)
node_10.add_child(leaf_19)
weight_69 = 0.3
weight_610 = 0.7
node_6.add_child(leaf_9, weight_69)
node_6.add_child(node_10, weight_610)
node_11 = ProductNode()
node_11.id = 11
leaf_20 = CategoricalSmoothedNode(var,
values)
leaf_20.id = 20
leaf_21 = CategoricalSmoothedNode(var,
values)
leaf_21.id = 21
node_11.add_child(leaf_20)
node_11.add_child(leaf_21)
node_12 = ProductNode()
node_12.id = 12
leaf_22 = CLTreeNode(vars=vars,
var_values=var_values,
data=s_data)
leaf_22.id = 22
leaf_23 = CategoricalSmoothedNode(var,
values)
leaf_23.id = 23
node_12.add_child(leaf_22)
node_12.add_child(leaf_23)
weight_711 = 0.5
weight_712 = 0.5
node_7.add_child(node_11, weight_711)
node_7.add_child(node_12, weight_712)
print('Added nodes')
root_node = SpnFactory.layered_pruned_linked_spn(node_1)
print('ROOT nODE', root_node)
spn = SpnFactory.layered_linked_spn(root_node)
print('SPN', spn)
assert spn.n_layers() == 3
for i, layer in enumerate(spn.top_down_layers()):
if i == 0:
assert layer.n_nodes() == 1
elif i == 1:
assert layer.n_nodes() == 4
elif i == 2:
assert layer.n_nodes() == 10
def test_sum_node_backprop():
# create child nodes
child1 = Node()
val1 = 1.
child1.set_val(val1)
child2 = Node()
val2 = 1.
child2.set_val(val2)
# create sum node and adding children to it
sum_node1 = SumNode()
weight11 = 0.8
weight12 = 0.2
sum_node1.add_child(child1, weight11)
sum_node1.add_child(child2, weight12)
# adding a coparent
sum_node2 = SumNode()
weight21 = 0.6
weight22 = 0.4
sum_node2.add_child(child1, weight21)
sum_node2.add_child(child2, weight22)
# evaluating
sum_node1.eval()
sum_node2.eval()
# setting the log derivatives to the parents
sum_node_der1 = 1.0
sum_node1.log_der = log(sum_node_der1)
sum_node1.backprop()
sum_node_der2 = 1.0
sum_node2.log_der = log(sum_node_der2)
sum_node2.backprop()
# checking for correctness
log_der1 = log(weight11 * sum_node_der1 +
weight21 * sum_node_der2)
log_der2 = log(weight12 * sum_node_der1 +
weight22 * sum_node_der2)
print('log ders 1:{lgd1} 2:{lgd2}'.format(lgd1=log_der1,
lgd2=log_der2))
assert_almost_equal(log_der1, child1.log_der, 15)
assert_almost_equal(log_der2, child2.log_der, 15)
# resetting
child1.log_der = LOG_ZERO
child2.log_der = LOG_ZERO
# now changing the initial der values
sum_node_der1 = 0.5
sum_node1.log_der = log(sum_node_der1)
sum_node1.backprop()
sum_node_der2 = 0.0
sum_node2.log_der = LOG_ZERO
sum_node2.backprop()
# checking for correctness
log_der1 = log(weight11 * sum_node_der1 +
weight21 * sum_node_der2)
log_der2 = log(weight12 * sum_node_der1 +
weight22 * sum_node_der2)
print('log ders 1:{lgd1} 2:{lgd2}'.format(lgd1=log_der1,
lgd2=log_der2))
assert_almost_equal(log_der1, child1.log_der, 15)
assert_almost_equal(log_der2, child2.log_der, 15)
def linked_kernel_density_estimation(cls,
n_instances,
features,
node_dict=None,
alpha=0.1
# ,batch_size=1,
# sparse=False
):
"""
WRITEME
"""
n_features = len(features)
# the top one is a sum layer with a single node
root_node = SumNode()
root_layer = SumLayerLinked([root_node])
# second one is a product layer with n_instances nodes
product_nodes = [ProductNode() for i in range(n_instances)]
product_layer = ProductLayerLinked(product_nodes)
# linking them to the root node
for prod_node in product_nodes:
root_node.add_child(prod_node, 1. / n_instances)
# last layer can be a categorical smoothed input
# or sum_layer + categorical indicator input
input_layer = None
layers = None
n_leaf_nodes = n_features * n_instances
if node_dict is None:
# creating a sum_layer with n_leaf_nodes
sum_nodes = [SumNode() for i in range(n_leaf_nodes)]
# store them into a layer
sum_layer = SumLayerLinked(sum_nodes)
# linking them to the products above
for i, prod_node in enumerate(product_nodes):
for j in range(n_features):
# getting the next n_features nodes
prod_node.add_child(sum_nodes[i * n_features + j])
# now creating the indicator nodes
input_layer = \
CategoricalIndicatorLayerLinked(vars=features)
# linking the sum nodes to the indicator vars
for i, sum_node in enumerate(sum_nodes):
# getting the feature id
j = i % n_features
# and thus its number of values
n_values = features[j]
# getting the indices of indicators
start_index = sum(features[:j])
end_index = start_index + n_values
indicators = [node for node in input_layer.nodes()
][start_index:end_index]
for ind_node in indicators:
sum_node.add_child(ind_node, 1. / n_values)
# storing levels
layers = [sum_layer, product_layer, root_layer]
else:
# create a categorical smoothed layer
input_layer = \
CategoricalSmoothedLayerLinked(vars=features,
node_dicts=node_dict,
alpha=alpha)
# it shall contain n_leaf_nodes nodes
smooth_nodes = list(input_layer.nodes())
assert len(smooth_nodes) == n_leaf_nodes
# linking it
for i, prod_node in enumerate(product_nodes):
for j in range(n_features):
# getting the next n_features nodes
prod_node.add_child(smooth_nodes[i * n_features + j])
# setting the used levels
layers = [product_layer, root_layer]
# create the spn from levels
kern_spn = SpnLinked(input_layer, layers)
return kern_spn
def test_build_linked_spn_from_scope_graph():
#
# creating a region graph as an input scope graph
n_cols = 2
n_rows = 2
coarse = 2
#
# create initial region
root_region = Region.create_whole_region(n_rows, n_cols)
region_graph = create_poon_region_graph(root_region, coarse=coarse)
# print(region_graph)
print('# partitions', region_graph.n_partitions())
print('# regions', region_graph.n_scopes())
print(region_graph)
#
#
k = 2
spn = build_linked_spn_from_scope_graph(region_graph, k)
print(spn)
print(spn.stats())
#
# back to the scope graph
root_layer = list(spn.root_layer().nodes())
assert len(root_layer) == 1
root = root_layer[0]
scope_graph = get_scope_graph_from_linked_spn(root)
print(scope_graph)
assert scope_graph == region_graph
#
# building an spn from scratch
#
# building leaf nodes
n_vars = 4
vars = [0, 1, 2, 3]
leaves = [
CategoricalIndicatorNode(var, val) for var in range(n_vars)
for val in [0, 1]
]
input_layer = CategoricalIndicatorLayer(nodes=leaves, vars=vars)
#
# building root
root_node = SumNode(var_scope=frozenset(vars))
root_layer = SumLayer([root_node])
#
# building product nodes
prod_list_1 = [ProductNode(var_scope=vars) for i in range(4)]
prod_list_2 = [ProductNode(var_scope=vars) for i in range(4)]
prod_nodes_1 = prod_list_1 + prod_list_2
product_layer_1 = ProductLayer(prod_nodes_1)
for p in prod_nodes_1:
root_node.add_child(p, 1.0 / len(prod_nodes_1))
#
# build sum nodes
sum_list_1 = [SumNode() for i in range(2)]
sum_list_2 = [SumNode() for i in range(2)]
sum_list_3 = [SumNode() for i in range(2)]
sum_list_4 = [SumNode() for i in range(2)]
sum_layer_2 = SumLayer(sum_list_1 + sum_list_2 + sum_list_3 + sum_list_4)
sum_pairs = []
for s_1 in sum_list_1:
for s_2 in sum_list_2:
sum_pairs.append((s_1, s_2))
for p, (s_1, s_2) in zip(prod_list_1, sum_pairs):
p.add_child(s_1)
p.add_child(s_2)
sum_pairs = []
for s_3 in sum_list_3:
for s_4 in sum_list_4:
sum_pairs.append((s_3, s_4))
for p, (s_3, s_4) in zip(prod_list_2, sum_pairs):
p.add_child(s_3)
p.add_child(s_4)
#
# again product nodes
prod_list_3 = [ProductNode() for i in range(4)]
prod_list_4 = [ProductNode() for i in range(4)]
prod_list_5 = [ProductNode() for i in range(4)]
prod_list_6 = [ProductNode() for i in range(4)]
product_layer_3 = ProductLayer(prod_list_3 + prod_list_4 + prod_list_5 +
prod_list_6)
for s in sum_list_1:
for p in prod_list_3:
s.add_child(p, 1.0 / len(prod_list_3))
for s in sum_list_2:
for p in prod_list_4:
s.add_child(p, 1.0 / len(prod_list_4))
for s in sum_list_3:
for p in prod_list_5:
s.add_child(p, 1.0 / len(prod_list_5))
for s in sum_list_4:
for p in prod_list_6:
s.add_child(p, 1.0 / len(prod_list_6))
#
# build sum nodes
sum_list_5 = [SumNode() for i in range(2)]
sum_list_6 = [SumNode() for i in range(2)]
sum_list_7 = [SumNode() for i in range(2)]
sum_list_8 = [SumNode() for i in range(2)]
sum_layer_4 = SumLayer(sum_list_5 + sum_list_6 + sum_list_7 + sum_list_8)
sum_pairs = []
for s_5 in sum_list_5:
for s_7 in sum_list_7:
sum_pairs.append((s_5, s_7))
for p, (s_5, s_7) in zip(prod_list_3, sum_pairs):
p.add_child(s_5)
p.add_child(s_7)
sum_pairs = []
for s_6 in sum_list_6:
for s_8 in sum_list_8:
sum_pairs.append((s_6, s_8))
for p, (s_6, s_8) in zip(prod_list_4, sum_pairs):
p.add_child(s_6)
p.add_child(s_8)
sum_pairs = []
for s_5 in sum_list_5:
for s_6 in sum_list_6:
sum_pairs.append((s_5, s_6))
for p, (s_5, s_6) in zip(prod_list_5, sum_pairs):
p.add_child(s_5)
p.add_child(s_6)
sum_pairs = []
for s_7 in sum_list_7:
for s_8 in sum_list_8:
sum_pairs.append((s_7, s_8))
for p, (s_7, s_8) in zip(prod_list_6, sum_pairs):
p.add_child(s_7)
p.add_child(s_8)
#
# linking to input layer
for s in sum_list_5:
for i in leaves[0:2]:
s.add_child(i, 0.5)
for s in sum_list_6:
for i in leaves[2:4]:
s.add_child(i, 0.5)
for s in sum_list_7:
for i in leaves[4:6]:
s.add_child(i, 0.5)
for s in sum_list_8:
for i in leaves[6:]:
s.add_child(i, 0.5)
lspn = LinkedSpn(input_layer=input_layer,
layers=[
sum_layer_4, product_layer_3, sum_layer_2,
product_layer_1, root_layer
])
print(lspn)
print(lspn.stats())
#
# trying to evaluate them
input_vec = numpy.array([[1., 1., 1., 0.], [0., 0., 0., 0.],
[0., 1., 1., 0.],
[MARG_IND, MARG_IND, MARG_IND, MARG_IND]]).T
res = spn.eval(input_vec)
print('First evaluation')
print(res)
res = lspn.eval(input_vec)
print('Second evaluation')
print(res)