def test_sum_node_is_complete(): # create a sum node with a scope scope = frozenset({0, 2, 7, 13}) sum_node = SumNode(var_scope=scope) # creating children with same scope children = [ProductNode(var_scope=scope) for i in range(4)] for prod_node in children: sum_node.add_child(prod_node, 1.0) assert sum_node.is_complete() # now altering one child's scope with one less var children[0].var_scope = frozenset({0, 7, 13}) assert sum_node.is_complete() is False # now adding one more children[0].var_scope = scope children[3].var_scope = frozenset({0, 2, 7, 13, 3}) assert not sum_node.is_complete() # now checking with indicator input nodes var = 4 sum_node = SumNode(var_scope=frozenset({var})) children = [CategoricalIndicatorNode(var=var, var_val=i) for i in range(4)] for input_node in children: sum_node.add_child(input_node, 1.0) assert sum_node.is_complete()
def test_spn_set_get_weights(): # create a simple spn root_node = SumNode() root_layer = SumLayer([root_node]) prod_node_1 = ProductNode() prod_node_2 = ProductNode() root_node.add_child(prod_node_1, 0.5) root_node.add_child(prod_node_2, 0.5) prod_layer = ProductLayer([prod_node_1, prod_node_2]) sum_node_1 = SumNode() sum_node_2 = SumNode() sum_node_3 = SumNode() prod_node_1.add_child(sum_node_1) prod_node_1.add_child(sum_node_2) prod_node_2.add_child(sum_node_2) prod_node_2.add_child(sum_node_3) sum_layer = SumLayer([sum_node_1, sum_node_2, sum_node_3]) ind_node_1 = CategoricalIndicatorNode(var=0, var_val=1) ind_node_2 = CategoricalIndicatorNode(var=0, var_val=1) ind_node_3 = CategoricalIndicatorNode(var=0, var_val=1) ind_node_4 = CategoricalIndicatorNode(var=0, var_val=1) ind_node_5 = CategoricalIndicatorNode(var=0, var_val=1) input_layer = CategoricalInputLayer( nodes=[ind_node_1, ind_node_2, ind_node_3, ind_node_4, ind_node_5]) sum_node_1.add_child(ind_node_1, 0.2) sum_node_1.add_child(ind_node_2, 0.2) sum_node_2.add_child(ind_node_2, 0.2) sum_node_2.add_child(ind_node_3, 0.2) sum_node_2.add_child(ind_node_4, 0.2) sum_node_3.add_child(ind_node_4, 0.2) sum_node_3.add_child(ind_node_5, 0.2) spn = Spn(input_layer=input_layer, layers=[sum_layer, prod_layer, root_layer]) print(spn) # storing these weights curr_weights = spn.get_weights() # setting the new weights spn.set_weights(weights_ds) # getting them again new_weights = spn.get_weights() # comparing them assert new_weights == weights_ds # now setting back the previous one spn.set_weights(curr_weights) # getting them back again old_weights = spn.get_weights() # and checking assert old_weights == curr_weights
def build_spn_layers(input_layer): # this is ugly... TODO try to beutify this process ind1 = input_layer._nodes[0] ind2 = input_layer._nodes[1] ind3 = input_layer._nodes[2] ind4 = input_layer._nodes[3] ind5 = input_layer._nodes[4] ind6 = input_layer._nodes[5] ind7 = input_layer._nodes[6] ind8 = input_layer._nodes[7] # creating sum nodes sum_node1 = SumNode() sum_node2 = SumNode() sum_node3 = SumNode() sum_node4 = SumNode() # linking them with nodes sum_node1.add_child(ind1, 0.5) sum_node1.add_child(ind2, 0.5) sum_node2.add_child(ind3, 0.1) sum_node2.add_child(ind4, 0.9) sum_node3.add_child(ind5, 0.3) sum_node3.add_child(ind6, 0.7) sum_node4.add_child(ind7, 0.6) sum_node4.add_child(ind8, 0.4) # creating sumlayer sum_layer = SumLayer([sum_node1, sum_node2, sum_node3, sum_node4]) # creating product nodes prod_node1 = ProductNode() prod_node2 = ProductNode() prod_node3 = ProductNode() # linking them to sum nodes prod_node1.add_child(sum_node1) prod_node1.add_child(sum_node2) prod_node2.add_child(sum_node2) prod_node2.add_child(sum_node3) prod_node3.add_child(sum_node3) prod_node3.add_child(sum_node4) # creating a product layer prod_layer = ProductLayer([prod_node1, prod_node2, prod_node3]) return sum_layer, prod_layer
def test_sum_node_create_and_eval(): # create child nodes child1 = Node() val1 = 1. child1.set_val(val1) child2 = Node() val2 = 1. child2.set_val(val2) # create sum node and adding children to it sum_node = SumNode() weight1 = 0.8 weight2 = 0.2 sum_node.add_child(child1, weight1) sum_node.add_child(child2, weight2) assert len(sum_node.children) == 2 assert len(sum_node.weights) == 2 assert len(sum_node.log_weights) == 2 log_weights = [log(weight1), log(weight2)] assert log_weights == sum_node.log_weights print(sum_node) # evaluating sum_node.eval() print(sum_node.log_val) assert_almost_equal(sum_node.log_val, log(val1 * weight1 + val2 * weight2), places=15) # changing values 1,0 val1 = 1. child1.set_val(val1) val2 = 0. child2.set_val(val2) # evaluating sum_node.eval() print(sum_node.log_val) assert_almost_equal(sum_node.log_val, log(val1 * weight1 + val2 * weight2), places=15) # changing values 0,0 -> LOG_ZERO val1 = 0. child1.set_val(val1) val2 = 0. child2.set_val(val2) # evaluating sum_node.eval() print(sum_node.log_val) assert_almost_equal(sum_node.log_val, LOG_ZERO, places=15)
def test_sum_node_create_and_eval_keras(): n_trials = 100 for i in range(n_trials): n_children = numpy.random.randint(1, 100) print('n children', n_children) children = [Node() for c in range(n_children)] weights = numpy.random.rand(n_children) weights = weights / weights.sum() # # create sum node and adding children to it sum_node = SumNode() for child, w in zip(children, weights): sum_node.add_child(child, w) child.log_vals = K.placeholder(ndim=2) assert len(sum_node.children) == n_children assert len(sum_node.weights) == n_children assert len(sum_node.log_weights) == n_children print(sum_node) # # evaluating for fake probabilities n_instances = numpy.random.randint(1, 100) print('n instances', n_instances) probs = numpy.random.rand(n_instances, n_children) # .astype(theano.config.floatX) log_probs = numpy.log(probs) log_vals = [] for d in range(n_instances): for c, child in enumerate(children): child.set_val(probs[d, c]) sum_node.eval() print('sum node eval') print(sum_node.log_val) log_vals.append(sum_node.log_val) # # now theano sum_node.build_k() eval_sum_node_f = K.function(inputs=[c.log_vals for c in children], outputs=[sum_node.log_vals]) keras_log_vals = eval_sum_node_f([log_probs[:, c].reshape(log_probs.shape[0], 1) for c in range(n_children)])[0] print(keras_log_vals) assert_array_almost_equal(numpy.array(log_vals).reshape(log_probs.shape[0], 1), keras_log_vals, decimal=4)
def test_sum_layer_is_complete(): # creating two scopes and two sum nodes scope1 = frozenset({0, 2, 3}) scope2 = frozenset({10}) sum_node_1 = SumNode(var_scope=scope1) sum_node_2 = SumNode(var_scope=scope2) # adding product nodes as children to the first, indicator the second for i in range(4): sum_node_1.add_child(ProductNode(var_scope=scope1), 1.0) sum_node_2.add_child(CategoricalIndicatorNode(var=10, var_val=i), 1.0) # creating sum layer sum_layer = SumLayer(nodes=[sum_node_1, sum_node_2]) assert sum_layer.is_complete() # now with errors in scope scope3 = frozenset({6}) sum_node_1 = SumNode(var_scope=scope1) sum_node_2 = SumNode(var_scope=scope3) # adding product nodes as children to the first, indicator the second for i in range(4): sum_node_1.add_child(ProductNode(var_scope=scope1), 1.0) sum_node_2.add_child(CategoricalIndicatorNode(var=10, var_val=i), 1.0) # creating sum layer sum_layer = SumLayer(nodes=[sum_node_1, sum_node_2]) assert not sum_layer.is_complete() sum_node_2.var_scope = scope2 assert sum_layer.is_complete() sum_node_2.children[3].var_scope = scope3 assert not sum_layer.is_complete()
def test_product_layer_is_decomposable(): # creating scopes and nodes scope1 = frozenset({0, 2, 3}) scope2 = frozenset({10, 9}) prod_node_1 = ProductNode(var_scope=scope1) prod_node_2 = ProductNode(var_scope=scope2) # creating children manually (argh=) for var in scope1: prod_node_1.add_child(SumNode(var_scope=frozenset({var}))) for var in scope2: prod_node_2.add_child(CategoricalSmoothedNode(var=var, var_values=2)) # creating layer prod_layer = ProductLayer(nodes=[prod_node_1, prod_node_2]) assert prod_layer.is_decomposable() # making it not decomposable anymore scope3 = frozenset({2}) prod_node_1.add_child(SumNode(var_scope=scope3)) assert not prod_layer.is_decomposable()
def estimate_kernel_density_spn(data_slice, feature_sizes, data, alpha, node_id_assoc, building_stack, slices_to_process): """ A mixture with one component for each instance """ instance_ids = data_slice.instance_ids feature_ids = data_slice.feature_ids current_id = data_slice.id n_instances = len(instance_ids) n_features = len(feature_ids) logging.info('Adding a kernel density estimation ' + 'over a slice {0} X {1}'.format(n_instances, n_features)) # # create sum node root_sum_node = SumNode(var_scope=frozenset(feature_ids)) data_slice.type = SumNode building_stack.append(data_slice) root_sum_node.id = current_id node_id_assoc[current_id] = root_sum_node # # for each instance for i in instance_ids: # # create a slice instance_slice = DataSlice(numpy.array([i]), feature_ids) slices_to_process.append(instance_slice) # # linking with appropriate weight data_slice.add_child(instance_slice, 1.0 / n_instances) return root_sum_node, node_id_assoc, building_stack, slices_to_process
def test_sum_node_normalize(): # create child nodes child1 = Node() val1 = 1. child1.set_val(val1) child2 = Node() val2 = 1. child2.set_val(val2) # create sum node and adding children to it sum_node = SumNode() weight1 = 1. weight2 = 0.2 weights = [weight1, weight2] sum_node.add_child(child1, weight1) sum_node.add_child(child2, weight2) un_sum = sum(weights) # normalizing sum_node.normalize() assert len(sum_node.children) == 2 assert len(sum_node.weights) == 2 assert len(sum_node.log_weights) == 2 # checking weight sum w_sum = sum(sum_node.weights) assert w_sum == 1. # and check the correct values normal_sum = [weight / un_sum for weight in weights] print(normal_sum) assert normal_sum == sum_node.weights # checking log_weights log_weights = [log(weight) for weight in normal_sum] print(log_weights) assert log_weights == sum_node.log_weights
def test_mini_spn_fit_em(): vars = numpy.array([2, 2, 2, 2]) input_layer = CategoricalIndicatorLayer(vars=vars) print(input_layer) ind1 = input_layer._nodes[0] ind2 = input_layer._nodes[1] ind3 = input_layer._nodes[2] ind4 = input_layer._nodes[3] ind5 = input_layer._nodes[4] ind6 = input_layer._nodes[5] ind7 = input_layer._nodes[6] ind8 = input_layer._nodes[7] # creating a sum layer of 4 nodes sum1 = SumNode() sum2 = SumNode() sum3 = SumNode() sum4 = SumNode() sum1.add_child(ind1, 0.6) sum1.add_child(ind2, 0.4) sum2.add_child(ind3, 0.5) sum2.add_child(ind4, 0.5) sum3.add_child(ind5, 0.7) sum3.add_child(ind6, 0.3) sum4.add_child(ind7, 0.4) sum4.add_child(ind8, 0.6) sum_layer = SumLayer(nodes=[sum1, sum2, sum3, sum4]) # and a top layer of 3 products prod1 = ProductNode() prod2 = ProductNode() prod3 = ProductNode() prod1.add_child(sum1) prod1.add_child(sum2) prod2.add_child(sum2) prod2.add_child(sum3) prod3.add_child(sum3) prod3.add_child(sum4) prod_layer = ProductLayer(nodes=[prod1, prod2, prod3]) # root layer root = SumNode() root.add_child(prod1, 0.4) root.add_child(prod2, 0.25) root.add_child(prod3, 0.35) root_layer = SumLayer(nodes=[root]) spn = Spn(input_layer=input_layer, layers=[sum_layer, prod_layer, root_layer]) print(spn) # training on obs spn.fit_em(train=syn_train_data, valid=syn_val_data, test=None, hard=True)
def create_valid_toy_spn(): # root layer whole_scope = frozenset({0, 1, 2, 3}) root_node = SumNode(var_scope=whole_scope) root_layer = SumLayer([root_node]) # prod layer prod_node_1 = ProductNode(var_scope=whole_scope) prod_node_2 = ProductNode(var_scope=whole_scope) prod_layer_1 = ProductLayer([prod_node_1, prod_node_2]) root_node.add_child(prod_node_1, 0.5) root_node.add_child(prod_node_2, 0.5) # sum layer scope_1 = frozenset({0, 1}) scope_2 = frozenset({2}) scope_3 = frozenset({3}) scope_4 = frozenset({2, 3}) sum_node_1 = SumNode(var_scope=scope_1) sum_node_2 = SumNode(var_scope=scope_2) sum_node_3 = SumNode(var_scope=scope_3) sum_node_4 = SumNode(var_scope=scope_4) prod_node_1.add_child(sum_node_1) prod_node_1.add_child(sum_node_2) prod_node_1.add_child(sum_node_3) prod_node_2.add_child(sum_node_1) prod_node_2.add_child(sum_node_4) sum_layer_1 = SumLayer([sum_node_1, sum_node_2, sum_node_3, sum_node_4]) # another product layer prod_node_3 = ProductNode(var_scope=scope_1) prod_node_4 = ProductNode(var_scope=scope_1) prod_node_5 = ProductNode(var_scope=scope_4) prod_node_6 = ProductNode(var_scope=scope_4) sum_node_1.add_child(prod_node_3, 0.5) sum_node_1.add_child(prod_node_4, 0.5) sum_node_4.add_child(prod_node_5, 0.5) sum_node_4.add_child(prod_node_6, 0.5) prod_layer_2 = ProductLayer([prod_node_3, prod_node_4, prod_node_5, prod_node_6]) # last sum one scope_5 = frozenset({0}) scope_6 = frozenset({1}) sum_node_5 = SumNode(var_scope=scope_5) sum_node_6 = SumNode(var_scope=scope_6) sum_node_7 = SumNode(var_scope=scope_5) sum_node_8 = SumNode(var_scope=scope_6) sum_node_9 = SumNode(var_scope=scope_2) sum_node_10 = SumNode(var_scope=scope_3) sum_node_11 = SumNode(var_scope=scope_2) sum_node_12 = SumNode(var_scope=scope_3) prod_node_3.add_child(sum_node_5) prod_node_3.add_child(sum_node_6) prod_node_4.add_child(sum_node_7) prod_node_4.add_child(sum_node_8) prod_node_5.add_child(sum_node_9) prod_node_5.add_child(sum_node_10) prod_node_6.add_child(sum_node_11) prod_node_6.add_child(sum_node_12) sum_layer_2 = SumLayer([sum_node_5, sum_node_6, sum_node_7, sum_node_8, sum_node_9, sum_node_10, sum_node_11, sum_node_12]) # input layer vars = [2, 3, 2, 2] input_layer = CategoricalIndicatorLayer(vars=vars) last_sum_nodes = [sum_node_2, sum_node_3, sum_node_5, sum_node_6, sum_node_7, sum_node_8, sum_node_9, sum_node_10, sum_node_11, sum_node_12] for sum_node in last_sum_nodes: (var_scope,) = sum_node.var_scope for input_node in input_layer.nodes(): if input_node.var == var_scope: sum_node.add_child(input_node, 1.0) spn = Spn(input_layer=input_layer, layers=[sum_layer_2, prod_layer_2, sum_layer_1, prod_layer_1, root_layer]) # print(spn) return spn
def test_categorical_to_indicator_input_layer(): # # creating all the data slices # the slicing is a fake stub # rows = 5 # cols = 5 var_1 = 0 values_1 = 2 var_2 = 1 values_2 = 3 var_3 = 2 values_3 = 4 node_1 = SumNode() node_1.id = 1 node_2 = ProductNode() node_2.id = 2 node_3 = SumNode() node_3.id = 3 # adding first level weight_12 = 0.4 weight_13 = 0.6 node_1.add_child(node_2, weight_12) node_1.add_child(node_3, weight_13) node_4 = ProductNode() node_4.id = 4 leaf_5 = CategoricalSmoothedNode(var_1, values_1) leaf_5.id = 5 # not adding the slice to the stack node_2.add_child(node_4) node_2.add_child(leaf_5) node_6 = SumNode() node_6.id = 6 node_7 = SumNode() node_7.id = 7 weight_36 = 0.1 weight_37 = 0.9 node_3.add_child(node_6, weight_36) node_3.add_child(node_7, weight_37) node_8 = ProductNode() node_8.id = 8 leaf_15 = CategoricalSmoothedNode(var_2, values_2) leaf_15.id = 15 node_4.add_child(node_8) node_4.add_child(leaf_15) leaf_13 = CategoricalSmoothedNode(var_3, values_3) leaf_13.id = 13 leaf_14 = CategoricalSmoothedNode(var_1, values_1) leaf_14.id = 14 node_8.add_child(leaf_13) node_8.add_child(leaf_14) node_9 = ProductNode() node_9.id = 9 leaf_16 = CategoricalSmoothedNode(var_2, values_2) leaf_16.id = 16 leaf_17 = CategoricalSmoothedNode(var_3, values_3) leaf_17.id = 17 node_9.add_child(leaf_16) node_9.add_child(leaf_17) node_10 = ProductNode() node_10.id = 10 leaf_18 = CategoricalSmoothedNode(var_2, values_2) leaf_18.id = 18 leaf_19 = CategoricalSmoothedNode(var_2, values_2) leaf_19.id = 19 node_10.add_child(leaf_18) node_10.add_child(leaf_19) weight_69 = 0.3 weight_610 = 0.7 node_6.add_child(node_9, weight_69) node_6.add_child(node_10, weight_610) node_11 = ProductNode() node_11.id = 11 leaf_20 = CategoricalSmoothedNode(var_1, values_1) leaf_20.id = 20 leaf_21 = CategoricalSmoothedNode(var_3, values_3) leaf_21.id = 21 node_11.add_child(leaf_20) node_11.add_child(leaf_21) node_12 = ProductNode() node_12.id = 12 leaf_22 = CategoricalSmoothedNode(var_1, values_1) leaf_22.id = 22 leaf_23 = CategoricalSmoothedNode(var_3, values_3) leaf_23.id = 23 node_12.add_child(leaf_22) node_12.add_child(leaf_23) weight_711 = 0.5 weight_712 = 0.5 node_7.add_child(node_11, weight_711) node_7.add_child(node_12, weight_712) root_node = SpnFactory.layered_pruned_linked_spn(node_1) print('ROOT nODE', root_node) spn = SpnFactory.layered_linked_spn(root_node) print('SPN', spn) assert spn.n_layers() == 3 for i, layer in enumerate(spn.top_down_layers()): if i == 0: assert layer.n_nodes() == 1 elif i == 1: assert layer.n_nodes() == 5 elif i == 2: assert layer.n_nodes() == 12 # # changing input layer spn = linked_categorical_input_to_indicators(spn) print('Changed input layer to indicator variables') print(spn)
def linked_kernel_density_estimation(cls, n_instances, features, node_dict=None, alpha=0.1 # ,batch_size=1, # sparse=False ): """ WRITEME """ n_features = len(features) # the top one is a sum layer with a single node root_node = SumNode() root_layer = SumLayerLinked([root_node]) # second one is a product layer with n_instances nodes product_nodes = [ProductNode() for i in range(n_instances)] product_layer = ProductLayerLinked(product_nodes) # linking them to the root node for prod_node in product_nodes: root_node.add_child(prod_node, 1. / n_instances) # last layer can be a categorical smoothed input # or sum_layer + categorical indicator input input_layer = None layers = None n_leaf_nodes = n_features * n_instances if node_dict is None: # creating a sum_layer with n_leaf_nodes sum_nodes = [SumNode() for i in range(n_leaf_nodes)] # store them into a layer sum_layer = SumLayerLinked(sum_nodes) # linking them to the products above for i, prod_node in enumerate(product_nodes): for j in range(n_features): # getting the next n_features nodes prod_node.add_child(sum_nodes[i * n_features + j]) # now creating the indicator nodes input_layer = \ CategoricalIndicatorLayerLinked(vars=features) # linking the sum nodes to the indicator vars for i, sum_node in enumerate(sum_nodes): # getting the feature id j = i % n_features # and thus its number of values n_values = features[j] # getting the indices of indicators start_index = sum(features[:j]) end_index = start_index + n_values indicators = [node for node in input_layer.nodes() ][start_index:end_index] for ind_node in indicators: sum_node.add_child(ind_node, 1. / n_values) # storing levels layers = [sum_layer, product_layer, root_layer] else: # create a categorical smoothed layer input_layer = \ CategoricalSmoothedLayerLinked(vars=features, node_dicts=node_dict, alpha=alpha) # it shall contain n_leaf_nodes nodes smooth_nodes = list(input_layer.nodes()) assert len(smooth_nodes) == n_leaf_nodes # linking it for i, prod_node in enumerate(product_nodes): for j in range(n_features): # getting the next n_features nodes prod_node.add_child(smooth_nodes[i * n_features + j]) # setting the used levels layers = [product_layer, root_layer] # create the spn from levels kern_spn = SpnLinked(input_layer, layers) return kern_spn
def linked_random_spn_top_down(cls, vars, n_layers, n_max_children, n_scope_children, max_scope_split, merge_prob=0.5, rand_gen=None): """ WRITEME """ def cluster_scopes(scope_list): cluster_dict = {} for i, var in enumerate(scope_list): cluster_dict[var] += {i} return cluster_dict def cluster_set_scope(scope_list): return {scope for scope in scope_list} def link_leaf_to_input_layer(sum_leaf, scope_var, input_layer, rand_gen): for indicator_node in input_layer.nodes(): if indicator_node.var == scope_var: rand_weight = rand_gen.random() sum_leaf.add_child(indicator_node, rand_weight) # print(sum_leaf, indicator_node, rand_weight) # normalizing sum_leaf.normalize() # # creating a product layer # def build_product_layer(parent_layer, parent_scope_list, n_max_children, n_scope_children, input_layer, rand_gen): # grouping the scopes of the parents scope_clusters = cluster_set_scope(parent_scope_list) # for each scope add a fixed number of children children_lists = { scope: [ ProductNode(var_scope=scope) for i in range(n_scope_children) ] for scope in scope_clusters } # counting which node is used children_counts = { scope: [0 for i in range(n_scope_children)] for scope in scope_clusters } # now link those randomly to their parent for parent, scope in zip(parent_layer.nodes(), parent_scope_list): # only for nodes not becoming leaves if len(scope) > 1: # sampling at most n_max_children from those in the same # scope children_scope_list = children_lists[scope] sample_length = min(len(children_scope_list), n_max_children) sampled_ids = rand_gen.sample(range(n_scope_children), sample_length) sampled_children = [None for i in range(sample_length)] for i, id in enumerate(sampled_ids): # getting the sampled child sampled_children[i] = children_scope_list[id] # updating its counter children_counts[scope][id] += 1 for child in sampled_children: # parent is a sum layer, we must set a random weight rand_weight = rand_gen.random() parent.add_child(child, rand_weight) # we can now normalize it parent.normalize() else: # binding the node to the input layer (scope_var, ) = scope link_leaf_to_input_layer(parent, scope_var, input_layer, rand_gen) # pruning those children never used for scope in children_lists.keys(): children_scope_list = children_lists[scope] scope_counts = children_counts[scope] used_children = [ child for count, child in zip(scope_counts, children_scope_list) if count > 0 ] children_lists[scope] = used_children # creating the layer and new scopelist # print('children list val', children_lists.values()) children_list = [ child for child in itertools.chain.from_iterable( children_lists.values()) ] scope_list = [ key for key, child_list in children_lists.items() for elem in child_list ] # print('children list', children_list) # print('scope list', scope_list) prod_layer = ProductLayerLinked(children_list) return prod_layer, scope_list def build_sum_layer(parent_layer, parent_scope_list, rand_gen, max_scope_split=-1, merge_prob=0.5): # keeping track of leaves # leaf_props = [] scope_clusters = cluster_set_scope(parent_scope_list) # looping through all the parent nodes and their scopes # in order to decompose their scope dec_scope_list = [] for scope in parent_scope_list: # decomposing their scope into k random pieces k = len(scope) if 1 < max_scope_split <= len(scope): k = rand_gen.randint(2, max_scope_split) shuffled_scope = list(scope) rand_gen.shuffle(shuffled_scope) dec_scopes = [ frozenset(shuffled_scope[i::k]) for i in range(k) ] dec_scope_list.append(dec_scopes) # if a decomposed scope consists of only one var, generate a # leaf # leaves = [(parent, (dec_scope,)) # for dec_scope in dec_scopes if len(dec_scope) == 1] # leaf_props.extend(leaves) # generating a unique decomposition used_decs = {} children_list = [] scope_list = [] for parent, decs in zip(parent_layer.nodes(), dec_scope_list): merge_count = 0 for scope in decs: sum_node = None try: rand_perc = rand_gen.random() if (merge_count < len(decs) - 1 and rand_perc > merge_prob): sum_node = used_decs[scope] merge_count += 1 else: raise Exception() except: # create a node for it sum_node = SumNode(var_scope=scope) children_list.append(sum_node) scope_list.append(scope) used_decs[scope] = sum_node parent.add_child(sum_node) # unique_dec = {frozenset(dec) for dec in # itertools.chain.from_iterable(dec_scope_list)} # print('unique dec', unique_dec) # building a dict scope->child # children_dict = {scope: SumNode() for scope in unique_dec} # now linking parents to their children # for parent, scope in zip(parent_layer.nodes(), # parent_scope_list): # dec_scopes = dec_scope_list[scope] # for dec in dec_scopes: # retrieving children # adding it # parent.add_child(children_dict[dec]) # we already have the nodes and their scopes # children_list = [child for child in children_dict.values()] # scope_list = [scope for scope in children_dict.keys()] sum_layer = SumLayerLinked(nodes=children_list) return sum_layer, scope_list # if no generator is provided, create a new one if rand_gen is None: rand_gen = random.Random() # create input layer # _vars = [2, 3, 2, 2, 4] input_layer = CategoricalIndicatorLayerLinked(vars=vars) # create root layer full_scope = frozenset({i for i in range(len(vars))}) root = SumNode(var_scope=full_scope) root_layer = SumLayerLinked(nodes=[root]) last_layer = root_layer # create top scope list last_scope_list = [full_scope] layers = [root_layer] layer_count = 0 stop_building = False while not stop_building: # checking for early termination # this one leads to split product nodes into leaves if layer_count >= n_layers: print('Max level reached, trying to stop') max_scope_split = -1 # build a new layer alternating types if isinstance(last_layer, SumLayerLinked): print('Building product layer') last_layer, last_scope_list = \ build_product_layer(last_layer, last_scope_list, n_max_children, n_scope_children, input_layer, rand_gen) elif isinstance(last_layer, ProductLayerLinked): print('Building sum layer') last_layer, last_scope_list = \ build_sum_layer(last_layer, last_scope_list, rand_gen, max_scope_split, merge_prob) # testing for more nodes to expand if last_layer.n_nodes() == 0: print('Stop building') stop_building = True else: layers.append(last_layer) layer_count += 1 # checking for early termination # if not stop_building: # if isinstance(last_layer, ProductLayerLinked): # building a sum layer splitting everything into one # length scopes # last_sum_layer, last_scope_list = \ # build_sum_layer(last_layer, # last_scope_list, # rand_gen, # max_scope_split=-1) # then linking each node to the input layer # for sum_leaf, scope in zip(last_sum_layer.nodes(), # last_scope_list): # (scope_var,) = scope # link_leaf_to_input_layer(sum_leaf, # scope_var, # input_layer, # rand_gen) # elif isinstance(last_layer, SumLayerLinked): # pass # print('LAYERS ', len(layers), '\n') # for i, layer in enumerate(layers): # print('LAYER ', i) # print(layer) # print('\n') spn = SpnLinked(input_layer=input_layer, layers=layers[::-1]) # testing # scope_list = [ # frozenset({1, 3, 4}), frozenset({2, 0}), frozenset({1, 3, 4})] # sum_layer = SumLayerLinked(nodes=[SumNode(), SumNode(), SumNode()]) # prod_layer, scope_list = build_product_layer( # sum_layer, scope_list, 2, 3, input_layer, rand_gen) # sum_layer1, scope_list_2 = build_sum_layer(prod_layer, # scope_list, # rand_gen, # max_scope_split=2 # ) # prod_layer_2, scope_list_3 = build_product_layer(sum_layer1, # scope_list_2, # 2, # 3, # input_layer, # rand_gen) # create spn from layers # spn = SpnLinked(input_layer=input_layer, # layers=[prod_layer_2, sum_layer1, # prod_layer, sum_layer, root_layer]) return spn
def linked_categorical_input_to_indicators(spn, input_layer=None): """ Convertes a linked spn categorical input layer into an indicator one """ # # get child, parent relations for node relinking child_assoc = retrieve_children_parent_assoc(spn) # # get input layer cat_input_layer = spn.input_layer() assert isinstance(cat_input_layer, CategoricalSmoothedLayerLinked) # # one indicator node for each var value vars = cat_input_layer.vars() if not vars: vars = list(sorted({node.var for node in cat_input_layer.nodes()})) feature_values = cat_input_layer.feature_vals() # print('vars', vars) # print('feature values', feature_values) indicator_nodes = [ CategoricalIndicatorNode(var, val) for i, var in enumerate(vars) for val in range(feature_values[i]) ] # for node in indicator_nodes: # print(node) indicator_map = defaultdict(set) for ind_node in indicator_nodes: indicator_map[ind_node.var].add(ind_node) sum_nodes = [] # # as many sum nodes as cat nodes for node in cat_input_layer.nodes(): sum_node = SumNode(var_scope=frozenset([node.var])) sum_nodes.append(sum_node) for ind_node in sorted(indicator_map[node.var], key=lambda x: x.var_val): sum_node.add_child(ind_node, numpy.exp(node._var_probs[ind_node.var_val])) # # removing links to parents parents = child_assoc[node] for p_node in parents: # # assume it to be a product node # TODO: generalize assert isinstance(p_node, ProductNode) p_node.children.remove(node) p_node.add_child(sum_node) # # creating layer sum_layer = SumLayerLinked(sum_nodes) indicator_layer = CategoricalIndicatorLayerLinked(indicator_nodes) cat_input_layer.disconnect_layer() spn.set_input_layer(indicator_layer) spn.insert_layer(sum_layer, 0) return spn
def test_pruned_spn_from_slices(): # # creating all the data slices # the slicing is a fake stub rows = 5 cols = 5 var = 1 values = 2 node_assoc = {} building_stack = deque() slice_1 = DataSlice.whole_slice(rows, cols) slice_1.type = SumNode node_1 = SumNode() node_1.id = slice_1.id node_assoc[node_1.id] = node_1 building_stack.append(slice_1) slice_2 = DataSlice.whole_slice(rows, cols) slice_2.type = ProductNode node_2 = ProductNode() node_2.id = slice_2.id node_assoc[node_2.id] = node_2 building_stack.append(slice_2) slice_3 = DataSlice.whole_slice(rows, cols) slice_3.type = SumNode node_3 = SumNode() node_3.id = slice_3.id node_assoc[node_3.id] = node_3 building_stack.append(slice_3) # adding first level slice_1.add_child(slice_2, 0.8) slice_1.add_child(slice_3, 0.2) slice_4 = DataSlice.whole_slice(rows, cols) slice_4.type = ProductNode node_4 = ProductNode() node_4.id = slice_4.id node_assoc[node_4.id] = node_4 building_stack.append(slice_4) leaf_5 = CategoricalSmoothedNode(var, values) slice_5 = DataSlice.whole_slice(rows, cols) leaf_5.id = slice_5.id node_assoc[leaf_5.id] = leaf_5 # not adding the slice to the stack slice_2.add_child(slice_4) slice_2.add_child(slice_5) slice_6 = DataSlice.whole_slice(rows, cols) slice_6.type = SumNode node_6 = SumNode() node_6.id = slice_6.id node_assoc[node_6.id] = node_6 building_stack.append(slice_6) slice_7 = DataSlice.whole_slice(rows, cols) slice_7.type = SumNode node_7 = SumNode() node_7.id = slice_7.id node_assoc[node_7.id] = node_7 building_stack.append(slice_7) slice_3.add_child(slice_6, 0.4) slice_3.add_child(slice_7, 0.6) slice_8 = DataSlice.whole_slice(rows, cols) slice_8.type = ProductNode node_8 = ProductNode() node_8.id = slice_8.id node_assoc[node_8.id] = node_8 building_stack.append(slice_8) leaf_15 = CategoricalSmoothedNode(var, values) slice_15 = DataSlice.whole_slice(rows, cols) leaf_15.id = slice_15.id node_assoc[leaf_15.id] = leaf_15 slice_4.add_child(slice_8) slice_4.add_child(slice_15) leaf_13 = CategoricalSmoothedNode(var, values) slice_13 = DataSlice.whole_slice(rows, cols) leaf_13.id = slice_13.id node_assoc[leaf_13.id] = leaf_13 leaf_14 = CategoricalSmoothedNode(var, values) slice_14 = DataSlice.whole_slice(rows, cols) leaf_14.id = slice_14.id node_assoc[leaf_14.id] = leaf_14 slice_8.add_child(slice_13) slice_8.add_child(slice_14) slice_9 = DataSlice.whole_slice(rows, cols) slice_9.type = ProductNode node_9 = ProductNode() node_9.id = slice_9.id node_assoc[node_9.id] = node_9 building_stack.append(slice_9) leaf_16 = CategoricalSmoothedNode(var, values) slice_16 = DataSlice.whole_slice(rows, cols) leaf_16.id = slice_16.id node_assoc[leaf_16.id] = leaf_16 leaf_17 = CategoricalSmoothedNode(var, values) slice_17 = DataSlice.whole_slice(rows, cols) leaf_17.id = slice_17.id node_assoc[leaf_17.id] = leaf_17 slice_9.add_child(slice_16) slice_9.add_child(slice_17) slice_10 = DataSlice.whole_slice(rows, cols) slice_10.type = ProductNode node_10 = ProductNode() node_10.id = slice_10.id node_assoc[node_10.id] = node_10 building_stack.append(slice_10) leaf_18 = CategoricalSmoothedNode(var, values) slice_18 = DataSlice.whole_slice(rows, cols) leaf_18.id = slice_18.id node_assoc[leaf_18.id] = leaf_18 leaf_19 = CategoricalSmoothedNode(var, values) slice_19 = DataSlice.whole_slice(rows, cols) leaf_19.id = slice_19.id node_assoc[leaf_19.id] = leaf_19 slice_10.add_child(slice_18) slice_10.add_child(slice_19) slice_6.add_child(slice_9, 0.1) slice_6.add_child(slice_10, 0.9) slice_11 = DataSlice.whole_slice(rows, cols) slice_11.type = ProductNode node_11 = ProductNode() node_11.id = slice_11.id node_assoc[node_11.id] = node_11 building_stack.append(slice_11) leaf_20 = CategoricalSmoothedNode(var, values) slice_20 = DataSlice.whole_slice(rows, cols) leaf_20.id = slice_20.id node_assoc[leaf_20.id] = leaf_20 leaf_21 = CategoricalSmoothedNode(var, values) slice_21 = DataSlice.whole_slice(rows, cols) leaf_21.id = slice_21.id node_assoc[leaf_21.id] = leaf_21 slice_11.add_child(slice_20) slice_11.add_child(slice_21) slice_12 = DataSlice.whole_slice(rows, cols) slice_12.type = ProductNode node_12 = ProductNode() node_12.id = slice_12.id node_assoc[node_12.id] = node_12 building_stack.append(slice_12) leaf_22 = CategoricalSmoothedNode(var, values) slice_22 = DataSlice.whole_slice(rows, cols) leaf_22.id = slice_22.id node_assoc[leaf_22.id] = leaf_22 leaf_23 = CategoricalSmoothedNode(var, values) slice_23 = DataSlice.whole_slice(rows, cols) leaf_23.id = slice_23.id node_assoc[leaf_23.id] = leaf_23 slice_12.add_child(slice_22) slice_12.add_child(slice_23) slice_7.add_child(slice_11, 0.2) slice_7.add_child(slice_12, 0.7) root_node = SpnFactory.pruned_spn_from_slices(node_assoc, building_stack) print('ROOT nODE', root_node) spn = SpnFactory.layered_linked_spn(root_node) print('SPN', spn) assert spn.n_layers() == 3 for i, layer in enumerate(spn.top_down_layers()): if i == 0: assert layer.n_nodes() == 1 elif i == 1: assert layer.n_nodes() == 5 elif i == 2: assert layer.n_nodes() == 12
def test_sum_node_backprop(): # create child nodes child1 = Node() val1 = 1. child1.set_val(val1) child2 = Node() val2 = 1. child2.set_val(val2) # create sum node and adding children to it sum_node1 = SumNode() weight11 = 0.8 weight12 = 0.2 sum_node1.add_child(child1, weight11) sum_node1.add_child(child2, weight12) # adding a coparent sum_node2 = SumNode() weight21 = 0.6 weight22 = 0.4 sum_node2.add_child(child1, weight21) sum_node2.add_child(child2, weight22) # evaluating sum_node1.eval() sum_node2.eval() # setting the log derivatives to the parents sum_node_der1 = 1.0 sum_node1.log_der = log(sum_node_der1) sum_node1.backprop() sum_node_der2 = 1.0 sum_node2.log_der = log(sum_node_der2) sum_node2.backprop() # checking for correctness log_der1 = log(weight11 * sum_node_der1 + weight21 * sum_node_der2) log_der2 = log(weight12 * sum_node_der1 + weight22 * sum_node_der2) print('log ders 1:{lgd1} 2:{lgd2}'.format(lgd1=log_der1, lgd2=log_der2)) assert_almost_equal(log_der1, child1.log_der, 15) assert_almost_equal(log_der2, child2.log_der, 15) # resetting child1.log_der = LOG_ZERO child2.log_der = LOG_ZERO # now changing the initial der values sum_node_der1 = 0.5 sum_node1.log_der = log(sum_node_der1) sum_node1.backprop() sum_node_der2 = 0.0 sum_node2.log_der = LOG_ZERO sum_node2.backprop() # checking for correctness log_der1 = log(weight11 * sum_node_der1 + weight21 * sum_node_der2) log_der2 = log(weight12 * sum_node_der1 + weight22 * sum_node_der2) print('log ders 1:{lgd1} 2:{lgd2}'.format(lgd1=log_der1, lgd2=log_der2)) assert_almost_equal(log_der1, child1.log_der, 15) assert_almost_equal(log_der2, child2.log_der, 15)
def fit_structure(self, data, feature_sizes): """ data is a numpy array of size {n_instances X n_features} feature_sizes is an array of integers representing feature ranges """ # # resetting the data slice ids (just in case) DataSlice.reset_id_counter() tot_n_instances = data.shape[0] tot_n_features = data.shape[1] logging.info('Learning SPN structure on a (%d X %d) dataset', tot_n_instances, tot_n_features) learn_start_t = perf_counter() # # a queue containing the data slices to process slices_to_process = deque() # a stack for building nodes building_stack = deque() # a dict to keep track of id->nodes node_id_assoc = {} # creating the first slice whole_slice = DataSlice.whole_slice(tot_n_instances, tot_n_features) slices_to_process.append(whole_slice) first_run = True # # iteratively process & split slices # while slices_to_process: # process a slice current_slice = slices_to_process.popleft() # pointers to the current data slice current_instances = current_slice.instance_ids current_features = current_slice.feature_ids current_id = current_slice.id n_instances = len(current_instances) n_features = len(current_features) logging.info('\n*** Processing slice %d (%d X %d)', current_id, n_instances, n_features) logging.debug('\tinstances:%s\n\tfeatures:%s', current_instances, current_features) # # is this a leaf node or we can split? if n_features == 1: logging.info('---> Adding a leaf (just one feature)') (feature_id, ) = current_features feature_size = feature_sizes[feature_id] # slicing from the original dataset slice_data_rows = data[current_instances, :] current_slice_data = slice_data_rows[:, current_features] # create the node leaf_node = CategoricalSmoothedNode( var=feature_id, var_values=feature_size, data=current_slice_data, instances=current_instances, alpha=self._alpha) # print('lnvf', leaf_node._var_freqs) # storing links # input_nodes.append(leaf_node) leaf_node.id = current_id node_id_assoc[current_id] = leaf_node logging.debug('\tCreated Smooth Node %s', leaf_node) elif (n_instances <= self._min_instances_slice and n_features > 1): # # splitting the slice on each feature logging.info('---> Few instances (%d), decompose all features', n_instances) # # shall put a cltree or if self._cltree_leaves: logging.info('into a Chow-Liu tree') # # slicing data slice_data_rows = data[current_instances, :] current_slice_data = slice_data_rows[:, current_features] current_feature_sizes = [ feature_sizes[i] for i in current_features ] # # creating a Chow-Liu tree as leaf leaf_node = CLTreeNode(vars=current_features, var_values=current_feature_sizes, data=current_slice_data, alpha=self._alpha) # # storing links leaf_node.id = current_id node_id_assoc[current_id] = leaf_node logging.debug('\tCreated Chow-Liu Tree Node %s', leaf_node) elif self._kde and n_instances > 1: estimate_kernel_density_spn(current_slice, feature_sizes, data, self._alpha, node_id_assoc, building_stack, slices_to_process) # elif n_instances == 1: # FIXME: there is a bug here else: current_slice, slices_to_process, building_stack, node_id_assoc = \ self.make_naive_factorization(current_slice, slices_to_process, building_stack, node_id_assoc) else: # # slicing from the original dataset slice_data_rows = data[current_instances, :] current_slice_data = slice_data_rows[:, current_features] split_on_features = False # # first run is a split on rows if first_run: logging.info('-- FIRST RUN --') first_run = False else: # # try clustering on cols # logging.debug('...trying to split on columns') split_start_t = perf_counter() print(data.shape) dependent_features, other_features = greedy_feature_split( data, current_slice, feature_sizes, self._g_factor, self._rand_gen) split_end_t = perf_counter() logging.info('...tried to split on columns in {}'.format( split_end_t - split_start_t)) if len(other_features) > 0: split_on_features = True # # have dependent components been found? if split_on_features: # # splitting on columns logging.info( '---> Splitting on features' + ' {} -> ({}, {})'.format(len(current_features), len(dependent_features), len(other_features))) # # creating two new data slices and putting them on queue first_slice = DataSlice(current_instances, dependent_features) second_slice = DataSlice(current_instances, other_features) slices_to_process.append(first_slice) slices_to_process.append(second_slice) children_ids = [first_slice.id, second_slice.id] # # storing link parent children current_slice.type = ProductNode building_stack.append(current_slice) current_slice.add_child(first_slice) current_slice.add_child(second_slice) # # creating product node prod_node = ProductNode( var_scope=frozenset(current_features)) prod_node.id = current_id node_id_assoc[current_id] = prod_node logging.debug('\tCreated Prod Node %s (with children %s)', prod_node, children_ids) else: # # clustering on rows logging.info('---> Splitting on rows') # # at most n_rows clusters, for sklearn k_row_clusters = min(self._n_cluster_splits, n_instances - 1) clustering = cluster_rows( data, current_slice, n_clusters=k_row_clusters, cluster_method=self._row_cluster_method, n_iters=self._n_iters, n_restarts=self._n_restarts, cluster_penalty=self._cluster_penalty, rand_gen=self._rand_gen, sklearn_args=self._sklearn_args) if len(clustering) < 2: logging.info('\n\n\nLess than 2 clusters\n\n (%d)', len(clustering)) logging.info('forcing a naive factorization') current_slice, slices_to_process, building_stack, node_id_assoc = \ self.make_naive_factorization(current_slice, slices_to_process, building_stack, node_id_assoc) else: # logging.debug('obtained clustering %s', clustering) logging.info('clustered into %d parts (min %d)', len(clustering), k_row_clusters) # splitting cluster_slices = [ DataSlice(cluster, current_features) for cluster in clustering ] cluster_slices_ids = [ slice.id for slice in cluster_slices ] # cluster_prior = 5.0 # cluster_weights = [(slice.n_instances() + cluster_prior) / # (n_instances + cluster_prior * len(cluster_slices)) # for slice in cluster_slices] cluster_weights = [ slice.n_instances() / n_instances for slice in cluster_slices ] # # appending for processing slices_to_process.extend(cluster_slices) # # storing links # current_slice.children = cluster_slices_ids # current_slice.weights = cluster_weights current_slice.type = SumNode building_stack.append(current_slice) for child_slice, child_weight in zip( cluster_slices, cluster_weights): current_slice.add_child(child_slice, child_weight) # # building a sum node SCOPES_DICT[frozenset(current_features)] += 1 sum_node = SumNode( var_scope=frozenset(current_features)) sum_node.id = current_id node_id_assoc[current_id] = sum_node logging.debug( '\tCreated Sum Node %s (with children %s)', sum_node, cluster_slices_ids) learn_end_t = perf_counter() logging.info('\n\n\tStructure learned in %f secs', (learn_end_t - learn_start_t)) # # linking the spn graph (parent -> children) # logging.info('===> Building tree') link_start_t = perf_counter() root_build_node = building_stack[0] root_node = node_id_assoc[root_build_node.id] logging.debug('root node: %s', root_node) root_node = SpnFactory.pruned_spn_from_slices(node_id_assoc, building_stack) link_end_t = perf_counter() logging.info('\tLinked the spn in %f secs (root_node %s)', (link_end_t - link_start_t), root_node) # # building layers # logging.info('===> Layering spn') layer_start_t = perf_counter() spn = SpnFactory.layered_linked_spn(root_node) layer_end_t = perf_counter() logging.info('\tLayered the spn in %f secs', (layer_end_t - layer_start_t)) logging.info('\nLearned SPN\n\n%s', spn.stats()) #logging.info('%s', SCOPES_DICT.most_common(30)) return spn
def build_linked_layered_spn(print_spn=True): # # building an indicator layer ind_x_00 = CategoricalIndicatorNode(0, 0) ind_x_01 = CategoricalIndicatorNode(0, 1) ind_x_10 = CategoricalIndicatorNode(1, 0) ind_x_11 = CategoricalIndicatorNode(1, 1) ind_x_20 = CategoricalIndicatorNode(2, 0) ind_x_21 = CategoricalIndicatorNode(2, 1) input_layer = CategoricalIndicatorLayer( [ind_x_00, ind_x_01, ind_x_10, ind_x_11, ind_x_20, ind_x_21]) # # sum layer # sum_node_1 = SumNode(frozenset([0])) sum_node_1.add_child(ind_x_00, 0.1) sum_node_1.add_child(ind_x_01, 0.9) sum_node_2 = SumNode(frozenset([0])) sum_node_2.add_child(ind_x_00, 0.4) sum_node_2.add_child(ind_x_01, 0.6) sum_node_3 = SumNode(frozenset([1])) sum_node_3.add_child(ind_x_10, 0.3) sum_node_3.add_child(ind_x_11, 0.7) sum_node_4 = SumNode(frozenset([1])) sum_node_4.add_child(ind_x_10, 0.6) sum_node_4.add_child(ind_x_11, 0.4) sum_node_5 = SumNode(frozenset([2])) sum_node_5.add_child(ind_x_20, 0.5) sum_node_5.add_child(ind_x_21, 0.5) sum_node_6 = SumNode(frozenset([2])) sum_node_6.add_child(ind_x_20, 0.2) sum_node_6.add_child(ind_x_21, 0.8) sum_layer_1 = SumLayerLinked([ sum_node_1, sum_node_2, sum_node_3, sum_node_4, sum_node_5, sum_node_6 ]) # # product nodes # # xy prod_node_7 = ProductNode(frozenset([0, 1])) prod_node_7.add_child(sum_node_1) prod_node_7.add_child(sum_node_3) prod_node_8 = ProductNode(frozenset([0, 1])) prod_node_8.add_child(sum_node_2) prod_node_8.add_child(sum_node_4) prod_node_9 = ProductNode(frozenset([0, 1])) prod_node_9.add_child(sum_node_1) prod_node_9.add_child(sum_node_3) # # yz prod_node_10 = ProductNode(frozenset([1, 2])) prod_node_10.add_child(sum_node_4) prod_node_10.add_child(sum_node_5) prod_node_11 = ProductNode(frozenset([1, 2])) prod_node_11.add_child(sum_node_4) prod_node_11.add_child(sum_node_6) prod_layer_2 = ProductLayerLinked( [prod_node_7, prod_node_8, prod_node_9, prod_node_10, prod_node_11]) # # sum nodes # # xy sum_node_12 = SumNode(frozenset([0, 1])) sum_node_12.add_child(prod_node_7, 0.1) sum_node_12.add_child(prod_node_8, 0.9) sum_node_13 = SumNode(frozenset([0, 1])) sum_node_13.add_child(prod_node_8, 0.7) sum_node_13.add_child(prod_node_9, 0.3) # # yz sum_node_14 = SumNode(frozenset([1, 2])) sum_node_14.add_child(prod_node_10, 0.6) sum_node_14.add_child(prod_node_11, 0.4) sum_layer_3 = SumLayerLinked([sum_node_12, sum_node_13, sum_node_14]) # # product nodes prod_node_15 = ProductNode(frozenset([0, 1, 2])) prod_node_15.add_child(sum_node_12) prod_node_15.add_child(sum_node_6) prod_node_16 = ProductNode(frozenset([0, 1, 2])) prod_node_16.add_child(sum_node_13) prod_node_16.add_child(sum_node_5) prod_node_17 = ProductNode(frozenset([0, 1, 2])) prod_node_17.add_child(sum_node_2) prod_node_17.add_child(sum_node_14) prod_layer_4 = ProductLayerLinked( [prod_node_15, prod_node_16, prod_node_17]) # # root sum_node_18 = SumNode(frozenset([0, 1, 2])) sum_node_18.add_child(prod_node_15, 0.2) sum_node_18.add_child(prod_node_16, 0.2) sum_node_18.add_child(prod_node_17, 0.6) sum_layer_5 = SumLayerLinked([sum_node_18]) # # creating the spn layers = [ sum_layer_1, prod_layer_2, sum_layer_3, prod_layer_4, sum_layer_5 ] nodes = [node for layer in layers for node in layer.nodes()] spn = SpnLinked(input_layer=input_layer, layers=layers) if print_spn: print(spn) return spn, layers, nodes
def test_sum_layer_create_and_eval(): # creating generic nodes node1 = Node() node2 = Node() node3 = Node() # whose values are val1 = 1. val2 = 1. val3 = 0. node1.set_val(val1) node2.set_val(val2) node3.set_val(val3) # setting weights weight11 = 0.2 weight12 = 0.3 weight13 = 0.5 weight21 = 0.3 weight22 = 0.7 weight32 = 0.4 weight33 = 0.6 # creating sum nodes sum1 = SumNode() sum2 = SumNode() sum3 = SumNode() # adding children sum1.add_child(node1, weight11) sum1.add_child(node2, weight12) sum1.add_child(node3, weight13) sum2.add_child(node1, weight21) sum2.add_child(node2, weight22) sum3.add_child(node2, weight32) sum3.add_child(node3, weight33) # adding to layer sum_layer = SumLayer([sum1, sum2, sum3]) # evaluation sum_layer.eval() # computing 'log values by hand' layer_evals = sum_layer.node_values() print('Layer eval nodes') print(layer_evals) logval1 = log(weight11 * val1 + weight12 * val2 + weight13 * val3) logval2 = log(weight21 * val1 + weight22 * val2) logval3 = log(weight32 * val2 + weight33 * val3) logvals = [logval1, logval2, logval3] print('log vals') print(logvals) # checking for correctness for logval, eval in zip(logvals, layer_evals): assert_almost_equal(logval, eval, PRECISION)
def test_sum_layer_backprop(): # input layer made of 5 generic nodes node1 = Node() node2 = Node() node3 = Node() node4 = Node() node5 = Node() # top layer made by 3 sum nodes sum1 = SumNode() sum2 = SumNode() sum3 = SumNode() # linking to input nodes weight11 = 0.3 sum1.add_child(node1, weight11) weight12 = 0.3 sum1.add_child(node2, weight12) weight13 = 0.4 sum1.add_child(node3, weight13) weight22 = 0.15 sum2.add_child(node2, weight22) weight23 = 0.15 sum2.add_child(node3, weight23) weight24 = 0.7 sum2.add_child(node4, weight24) weight33 = 0.4 sum3.add_child(node3, weight33) weight34 = 0.25 sum3.add_child(node4, weight34) weight35 = 0.35 sum3.add_child(node5, weight35) sum_layer = SumLayer([sum1, sum2, sum3]) # setting input values val1 = 0.0 node1.set_val(val1) val2 = 0.5 node2.set_val(val2) val3 = 0.3 node3.set_val(val3) val4 = 1.0 node4.set_val(val4) val5 = 0.0 node5.set_val(val5) # evaluating sum_layer.eval() print('eval\'d layer:', sum_layer.node_values()) # set the parent derivatives sum_der1 = 1.0 sum1.log_der = log(sum_der1) sum_der2 = 1.0 sum2.log_der = log(sum_der2) sum_der3 = 0.0 sum3.log_der = LOG_ZERO # back prop layer wise sum_layer.backprop() # check for correctness try: log_der1 = log(sum_der1 * weight11) except: log_der1 = LOG_ZERO try: log_der2 = log(sum_der1 * weight12 + sum_der2 * weight22) except: log_der2 = LOG_ZERO try: log_der3 = log(sum_der1 * weight13 + sum_der2 * weight23 + sum_der3 * weight33) except: log_der3 = LOG_ZERO try: log_der4 = log(sum_der2 * weight24 + sum_der3 * weight34) except: log_der4 = LOG_ZERO try: log_der5 = log(sum_der3 * weight35) except: log_der5 = LOG_ZERO # printing, just in case print('child log der', node1.log_der, node2.log_der, node3.log_der, node4.log_der, node5.log_der) print('exact log der', log_der1, log_der2, log_der3, log_der4, log_der5) if IS_LOG_ZERO(log_der1): assert IS_LOG_ZERO(node1.log_der) else: assert_almost_equal(log_der1, node1.log_der, 15) if IS_LOG_ZERO(log_der2): assert IS_LOG_ZERO(node2.log_der) else: assert_almost_equal(log_der2, node2.log_der, 15) if IS_LOG_ZERO(log_der3): assert IS_LOG_ZERO(node3.log_der) else: assert_almost_equal(log_der3, node3.log_der, 15) if IS_LOG_ZERO(log_der4): assert IS_LOG_ZERO(node4.log_der) else: assert_almost_equal(log_der4, node4.log_der, 15) if IS_LOG_ZERO(log_der5): assert IS_LOG_ZERO(node5.log_der) else: assert_almost_equal(log_der5, node5.log_der, 15) # updating weights eta = 0.1 sum_layer.update_weights(Spn.test_weight_update, 0) # checking for correctness weight_u11 = sum_der1 * val1 * eta + weight11 weight_u12 = sum_der1 * val2 * eta + weight12 weight_u13 = sum_der1 * val3 * eta + weight13 weight_u22 = sum_der2 * val2 * eta + weight22 weight_u23 = sum_der2 * val3 * eta + weight23 weight_u24 = sum_der2 * val4 * eta + weight24 weight_u33 = sum_der3 * val3 * eta + weight33 weight_u34 = sum_der3 * val4 * eta + weight34 weight_u35 = sum_der3 * val5 * eta + weight35 # normalizing weight_sum1 = weight_u11 + weight_u12 + weight_u13 weight_sum2 = weight_u22 + weight_u23 + weight_u24 weight_sum3 = weight_u33 + weight_u34 + weight_u35 weight_u11 = weight_u11 / weight_sum1 weight_u12 = weight_u12 / weight_sum1 weight_u13 = weight_u13 / weight_sum1 weight_u22 = weight_u22 / weight_sum2 weight_u23 = weight_u23 / weight_sum2 weight_u24 = weight_u24 / weight_sum2 weight_u33 = weight_u33 / weight_sum3 weight_u34 = weight_u34 / weight_sum3 weight_u35 = weight_u35 / weight_sum3 print('expected weights', weight_u11, weight_u12, weight_u13, weight_u22, weight_u23, weight_u24, weight_u33, weight_u34, weight_u35) print('found weights', sum1.weights[0], sum1.weights[1], sum1.weights[2], sum2.weights[0], sum2.weights[1], sum2.weights[2], sum3.weights[0], sum3.weights[1], sum3.weights[2]) assert_almost_equal(weight_u11, sum1.weights[0], 10) assert_almost_equal(weight_u12, sum1.weights[1], 10) assert_almost_equal(weight_u13, sum1.weights[2], 10) assert_almost_equal(weight_u22, sum2.weights[0], 10) assert_almost_equal(weight_u23, sum2.weights[1], 10) assert_almost_equal(weight_u24, sum2.weights[2], 10) assert_almost_equal(weight_u33, sum3.weights[0], 10) assert_almost_equal(weight_u34, sum3.weights[1], 10) assert_almost_equal(weight_u35, sum3.weights[2], 10) # # resetting derivatives # node1.log_der = LOG_ZERO node2.log_der = LOG_ZERO node3.log_der = LOG_ZERO node4.log_der = LOG_ZERO node5.log_der = LOG_ZERO # setting new values as inputs val1 = 0.0 node1.set_val(val1) val2 = 0.0 node2.set_val(val2) val3 = 0.3 node3.set_val(val3) val4 = 1.0 node4.set_val(val4) val5 = 1.0 node5.set_val(val5) # evaluating again sum_layer.eval() print('eval\'d layer:', sum_layer.node_values()) # set the parent derivatives sum_der1 = 1.0 sum1.log_der = log(sum_der1) sum_der2 = 1.0 sum2.log_der = log(sum_der2) sum_der3 = 0.0 sum3.log_der = LOG_ZERO # back prop layer wise sum_layer.backprop() # check for correctness try: log_der1 = log(sum_der1 * weight_u11) except: log_der1 = LOG_ZERO try: log_der2 = log(sum_der1 * weight_u12 + sum_der2 * weight_u22) except: log_der2 = LOG_ZERO try: log_der3 = log(sum_der1 * weight_u13 + sum_der2 * weight_u23 + sum_der3 * weight_u33) except: log_der3 = LOG_ZERO try: log_der4 = log(sum_der2 * weight_u24 + sum_der3 * weight_u34) except: log_der4 = LOG_ZERO try: log_der5 = log(sum_der3 * weight_u35) except: log_der5 = LOG_ZERO # printing, just in case print('child log der', node1.log_der, node2.log_der, node3.log_der, node4.log_der, node5.log_der) print('exact log der', log_der1, log_der2, log_der3, log_der4, log_der5) if IS_LOG_ZERO(log_der1): assert IS_LOG_ZERO(node1.log_der) else: assert_almost_equal(log_der1, node1.log_der, 15) if IS_LOG_ZERO(log_der2): assert IS_LOG_ZERO(node2.log_der) else: assert_almost_equal(log_der2, node2.log_der, 15) if IS_LOG_ZERO(log_der3): assert IS_LOG_ZERO(node3.log_der) else: assert_almost_equal(log_der3, node3.log_der, 15) if IS_LOG_ZERO(log_der4): assert IS_LOG_ZERO(node4.log_der) else: assert_almost_equal(log_der4, node4.log_der, 15) if IS_LOG_ZERO(log_der5): assert IS_LOG_ZERO(node5.log_der) else: assert_almost_equal(log_der5, node5.log_der, 15)
def test_layered_linked_spn(): # creating single nodes # this code is replicated TODO: make a function root = SumNode() prod1 = ProductNode() prod2 = ProductNode() prod3 = ProductNode() sum1 = SumNode() sum2 = SumNode() sum3 = SumNode() sum4 = SumNode() ind1 = CategoricalIndicatorNode(var=0, var_val=0) ind2 = CategoricalIndicatorNode(var=0, var_val=1) ind3 = CategoricalIndicatorNode(var=1, var_val=0) ind4 = CategoricalIndicatorNode(var=1, var_val=1) ind5 = CategoricalIndicatorNode(var=2, var_val=0) ind6 = CategoricalIndicatorNode(var=2, var_val=1) ind7 = CategoricalIndicatorNode(var=2, var_val=2) ind8 = CategoricalIndicatorNode(var=3, var_val=0) ind9 = CategoricalIndicatorNode(var=3, var_val=1) ind10 = CategoricalIndicatorNode(var=3, var_val=2) ind11 = CategoricalIndicatorNode(var=3, var_val=3) prod4 = ProductNode() prod5 = ProductNode() prod6 = ProductNode() prod7 = ProductNode() # linking nodes root.add_child(prod1, 0.3) root.add_child(prod2, 0.3) root.add_child(prod3, 0.4) prod1.add_child(sum1) prod1.add_child(sum2) prod2.add_child(ind7) prod2.add_child(ind8) prod2.add_child(ind11) prod3.add_child(sum3) prod3.add_child(sum4) sum1.add_child(ind1, 0.3) sum1.add_child(ind2, 0.3) sum1.add_child(prod4, 0.4) sum2.add_child(ind2, 0.5) sum2.add_child(prod4, 0.2) sum2.add_child(prod5, 0.3) sum3.add_child(prod6, 0.5) sum3.add_child(prod7, 0.5) sum4.add_child(prod6, 0.5) sum4.add_child(prod7, 0.5) prod4.add_child(ind3) prod4.add_child(ind4) prod5.add_child(ind5) prod5.add_child(ind6) prod6.add_child(ind9) prod6.add_child(ind10) prod7.add_child(ind9) prod7.add_child(ind10) spn = SpnFactory.layered_linked_spn(root) print(spn) print(spn.stats())
def test_product_node_is_decomposable(): # create a prod node with a scope scope = frozenset({0, 2, 7, 13}) # creating sub scopes sub_scope_1 = frozenset({0}) sub_scope_2 = frozenset({0, 2}) sub_scope_3 = frozenset({7}) sub_scope_4 = frozenset({17}) sub_scope_5 = frozenset({7, 13}) # now with decomposable children child1 = SumNode(var_scope=sub_scope_2) child2 = SumNode(var_scope=sub_scope_5) child3 = SumNode(var_scope=sub_scope_2) child4 = SumNode(var_scope=sub_scope_1) prod_node = ProductNode(var_scope=scope) prod_node.add_child(child1) prod_node.add_child(child2) assert prod_node.is_decomposable() prod_node = ProductNode(var_scope=scope) prod_node.add_child(child4) prod_node.add_child(child1) prod_node.add_child(child2) assert not prod_node.is_decomposable() prod_node = ProductNode(var_scope=scope) prod_node.add_child(child4) prod_node.add_child(child2) assert not prod_node.is_decomposable() # now with input nodes child5 = CategoricalSmoothedNode(var=0, var_values=2) child6 = CategoricalSmoothedNode(var=2, var_values=2) child7 = CategoricalSmoothedNode(var=7, var_values=2) child8 = CategoricalSmoothedNode(var=13, var_values=2) child9 = CategoricalSmoothedNode(var=17, var_values=2) prod_node = ProductNode(var_scope=scope) prod_node.add_child(child5) prod_node.add_child(child6) prod_node.add_child(child7) prod_node.add_child(child8) assert prod_node.is_decomposable() prod_node = ProductNode(var_scope=scope) prod_node.add_child(child5) prod_node.add_child(child6) prod_node.add_child(child7) prod_node.add_child(child9) assert not prod_node.is_decomposable() prod_node = ProductNode(var_scope=scope) prod_node.add_child(child5) prod_node.add_child(child6) prod_node.add_child(child8) assert not prod_node.is_decomposable()
def test_linked_to_theano_indicator(): # creating single nodes root = SumNode() prod1 = ProductNode() prod2 = ProductNode() prod3 = ProductNode() sum1 = SumNode() sum2 = SumNode() sum3 = SumNode() sum4 = SumNode() ind1 = CategoricalIndicatorNode(var=0, var_val=0) ind2 = CategoricalIndicatorNode(var=0, var_val=1) ind3 = CategoricalIndicatorNode(var=1, var_val=0) ind4 = CategoricalIndicatorNode(var=1, var_val=1) ind5 = CategoricalIndicatorNode(var=2, var_val=0) ind6 = CategoricalIndicatorNode(var=2, var_val=1) ind7 = CategoricalIndicatorNode(var=2, var_val=2) ind8 = CategoricalIndicatorNode(var=3, var_val=0) ind9 = CategoricalIndicatorNode(var=3, var_val=1) ind10 = CategoricalIndicatorNode(var=3, var_val=2) ind11 = CategoricalIndicatorNode(var=3, var_val=3) prod4 = ProductNode() prod5 = ProductNode() prod6 = ProductNode() prod7 = ProductNode() # linking nodes root.add_child(prod1, 0.3) root.add_child(prod2, 0.3) root.add_child(prod3, 0.4) prod1.add_child(sum1) prod1.add_child(sum2) prod2.add_child(ind7) prod2.add_child(ind8) prod2.add_child(ind11) prod3.add_child(sum3) prod3.add_child(sum4) sum1.add_child(ind1, 0.3) sum1.add_child(ind2, 0.3) sum1.add_child(prod4, 0.4) sum2.add_child(ind2, 0.5) sum2.add_child(prod4, 0.2) sum2.add_child(prod5, 0.3) sum3.add_child(prod6, 0.5) sum3.add_child(prod7, 0.5) sum4.add_child(prod6, 0.5) sum4.add_child(prod7, 0.5) prod4.add_child(ind3) prod4.add_child(ind4) prod5.add_child(ind5) prod5.add_child(ind6) prod6.add_child(ind9) prod6.add_child(ind10) prod7.add_child(ind9) prod7.add_child(ind10) # building layers from nodes root_layer = SumLayerLinked([root]) prod_layer = ProductLayerLinked([prod1, prod2, prod3]) sum_layer = SumLayerLinked([sum1, sum2, sum3, sum4]) aprod_layer = ProductLayerLinked([prod4, prod5, prod6, prod7]) ind_layer = CategoricalIndicatorLayer(nodes=[ ind1, ind2, ind3, ind4, ind5, ind6, ind7, ind8, ind9, ind10, ind11 ]) # creating the linked spn spn_linked = SpnLinked( input_layer=ind_layer, layers=[aprod_layer, sum_layer, prod_layer, root_layer]) print(spn_linked) # converting to theano repr spn_theano = SpnFactory.linked_to_theano(spn_linked) print(spn_theano) # time for some inference comparison for instance in I: print('linked') res_l = spn_linked.eval(instance) print(res_l) print('theano') res_t = spn_theano.eval(instance) print(res_t) assert_array_almost_equal(res_l, res_t)
def linked_naive_factorization(cls, features, node_dict=None, alpha=0.1): """ WRITEME """ n_features = len(features) # create an input layer input_layer = None layers = None # first layer is a product layer with n_feature children root_node = ProductNode() root_layer = ProductLayerLinked([root_node]) # second is a sum node on an indicator layer if node_dict is None: # creating sum nodes sum_nodes = [SumNode() for i in range(n_features)] # linking to the root for node in sum_nodes: root_node.add_child(node) # store into a level sum_layer = SumLayerLinked(sum_nodes) # now create an indicator layer input_layer = CategoricalIndicatorLayerLinked(vars=features) # and linking it # TODO make this a function for i, sum_node in enumerate(sum_nodes): # getting the feature id j = i % n_features # and thus its number of values n_values = features[j] # getting the indices of indicators start_index = sum(features[:j]) end_index = start_index + n_values indicators = [node for node in input_layer.nodes() ][start_index:end_index] for ind_node in indicators: sum_node.add_child(ind_node, 1. / n_values) # collecting layers layers = [sum_layer, root_layer] # or a categorical smoothed layer else: input_layer = CategoricalSmoothedLayerLinked(vars=features, node_dicts=node_dict, alpha=alpha) # it shall contain n_features nodes smooth_nodes = list(input_layer.nodes()) assert len(smooth_nodes) == n_features for node in smooth_nodes: root_node.add_child(node) # set layers accordingly layers = [root_layer] # build the spn naive_fact_spn = SpnLinked(input_layer, layers) return naive_fact_spn
def test_spn_backprop(): # create initial layer node1 = Node() node2 = Node() node3 = Node() node4 = Node() node5 = Node() input_layer = CategoricalInputLayer([node1, node2, node3, node4, node5]) # top layer made by 3 sum nodes sum1 = SumNode() sum2 = SumNode() sum3 = SumNode() # linking to input nodes weight11 = 0.3 sum1.add_child(node1, weight11) weight12 = 0.3 sum1.add_child(node2, weight12) weight13 = 0.4 sum1.add_child(node3, weight13) weight22 = 0.15 sum2.add_child(node2, weight22) weight23 = 0.15 sum2.add_child(node3, weight23) weight24 = 0.7 sum2.add_child(node4, weight24) weight33 = 0.4 sum3.add_child(node3, weight33) weight34 = 0.25 sum3.add_child(node4, weight34) weight35 = 0.35 sum3.add_child(node5, weight35) sum_layer = SumLayer([sum1, sum2, sum3]) # another layer with two product nodes prod1 = ProductNode() prod2 = ProductNode() prod1.add_child(sum1) prod1.add_child(sum2) prod2.add_child(sum2) prod2.add_child(sum3) prod_layer = ProductLayer([prod1, prod2]) # root layer, double sum root1 = SumNode() root2 = SumNode() weightr11 = 0.5 root1.add_child(prod1, weightr11) weightr12 = 0.5 root1.add_child(prod2, weightr12) weightr21 = 0.9 root2.add_child(prod1, weightr21) weightr22 = 0.1 root2.add_child(prod2, weightr22) root_layer = SumLayer([root1, root2]) # root_layer = SumLayer([root1]) # create the spn spn = Spn(input_layer=input_layer, layers=[sum_layer, prod_layer, root_layer]) # setting the input values val1 = 0.0 node1.set_val(val1) val2 = 0.5 node2.set_val(val2) val3 = 0.3 node3.set_val(val3) val4 = 1.0 node4.set_val(val4) val5 = 0.0 node5.set_val(val5) # evaluating the spn res = spn.test_eval() print('spn eval\'d', res) # backprop spn.backprop() # computing derivatives by hand # topdown: root layer root_der = 1.0 log_root_der = log(root_der) # print('root ders', root1.log_der, root2.log_der) print('root ders', root1.log_der) assert_almost_equal(log_root_der, root1.log_der) assert_almost_equal(log_root_der, root2.log_der) # product layer prod_der1 = (root_der * weightr11 + root_der * weightr21) prod_der2 = (root_der * weightr12 + root_der * weightr22) # prod_der1 = (root_der * weightr11) # prod_der2 = (root_der * weightr12) log_prod_der1 = log(prod_der1) if prod_der1 > 0.0 else LOG_ZERO log_prod_der2 = log(prod_der2) if prod_der2 > 0.0 else LOG_ZERO print('found prod ders', prod1.log_der, prod2.log_der) print('expect prod ders', log_prod_der1, log_prod_der2) if IS_LOG_ZERO(log_prod_der1): assert IS_LOG_ZERO(prod1.log_der) else: assert_almost_equal(log_prod_der1, prod1.log_der) if IS_LOG_ZERO(log_prod_der2): assert IS_LOG_ZERO(prod2.log_der) else: assert_almost_equal(log_prod_der2, prod2.log_der) # sum layer sum_der1 = ( prod_der1 * (weight22 * val2 + weight23 * val3 + weight24 * val4)) log_sum_der1 = log(sum_der1) if sum_der1 > 0.0 else LOG_ZERO sum_der2 = (prod_der1 * (weight11 * val1 + weight12 * val2 + weight13 * val3) + prod_der2 * (weight33 * val3 + weight34 * val4 + weight35 * val5)) log_sum_der2 = log(sum_der2) if sum_der2 > 0.0 else LOG_ZERO sum_der3 = (prod_der2 * (weight22 * val2 + weight23 * val3 + weight24 * val4)) log_sum_der3 = log(sum_der3) if sum_der3 > 0.0 else LOG_ZERO print('expected sum ders', log_sum_der1, log_sum_der2, log_sum_der3) print('found sum ders', sum1.log_der, sum2.log_der, sum3.log_der) if IS_LOG_ZERO(log_sum_der1): assert IS_LOG_ZERO(sum1.log_der) else: assert_almost_equal(log_sum_der1, sum1.log_der) if IS_LOG_ZERO(log_sum_der2): assert IS_LOG_ZERO(sum2.log_der) else: assert_almost_equal(log_sum_der2, sum2.log_der) if IS_LOG_ZERO(log_sum_der3): assert IS_LOG_ZERO(sum3.log_der) else: assert_almost_equal(log_sum_der3, sum3.log_der) # final level, the first one try: log_der1 = log(sum_der1 * weight11) except: log_der1 = LOG_ZERO try: log_der2 = log(sum_der1 * weight12 + sum_der2 * weight22) except: log_der2 = LOG_ZERO try: log_der3 = log(sum_der1 * weight13 + sum_der2 * weight23 + sum_der3 * weight33) except: log_der3 = LOG_ZERO try: log_der4 = log(sum_der2 * weight24 + sum_der3 * weight34) except: log_der4 = LOG_ZERO try: log_der5 = log(sum_der3 * weight35) except: log_der5 = LOG_ZERO # printing, just in case print('child log der', node1.log_der, node2.log_der, node3.log_der, node4.log_der, node5.log_der) print('exact log der', log_der1, log_der2, log_der3, log_der4, log_der5) if IS_LOG_ZERO(log_der1): assert IS_LOG_ZERO(node1.log_der) else: assert_almost_equal(log_der1, node1.log_der, 15) if IS_LOG_ZERO(log_der2): assert IS_LOG_ZERO(node2.log_der) else: assert_almost_equal(log_der2, node2.log_der, 15) if IS_LOG_ZERO(log_der3): assert IS_LOG_ZERO(node3.log_der) else: assert_almost_equal(log_der3, node3.log_der, 15) if IS_LOG_ZERO(log_der4): assert IS_LOG_ZERO(node4.log_der) else: assert_almost_equal(log_der4, node4.log_der, 15) if IS_LOG_ZERO(log_der5): assert IS_LOG_ZERO(node5.log_der) else: assert_almost_equal(log_der5, node5.log_der, 15)
def build_sum_layer(parent_layer, parent_scope_list, rand_gen, max_scope_split=-1, merge_prob=0.5): # keeping track of leaves # leaf_props = [] scope_clusters = cluster_set_scope(parent_scope_list) # looping through all the parent nodes and their scopes # in order to decompose their scope dec_scope_list = [] for scope in parent_scope_list: # decomposing their scope into k random pieces k = len(scope) if 1 < max_scope_split <= len(scope): k = rand_gen.randint(2, max_scope_split) shuffled_scope = list(scope) rand_gen.shuffle(shuffled_scope) dec_scopes = [ frozenset(shuffled_scope[i::k]) for i in range(k) ] dec_scope_list.append(dec_scopes) # if a decomposed scope consists of only one var, generate a # leaf # leaves = [(parent, (dec_scope,)) # for dec_scope in dec_scopes if len(dec_scope) == 1] # leaf_props.extend(leaves) # generating a unique decomposition used_decs = {} children_list = [] scope_list = [] for parent, decs in zip(parent_layer.nodes(), dec_scope_list): merge_count = 0 for scope in decs: sum_node = None try: rand_perc = rand_gen.random() if (merge_count < len(decs) - 1 and rand_perc > merge_prob): sum_node = used_decs[scope] merge_count += 1 else: raise Exception() except: # create a node for it sum_node = SumNode(var_scope=scope) children_list.append(sum_node) scope_list.append(scope) used_decs[scope] = sum_node parent.add_child(sum_node) # unique_dec = {frozenset(dec) for dec in # itertools.chain.from_iterable(dec_scope_list)} # print('unique dec', unique_dec) # building a dict scope->child # children_dict = {scope: SumNode() for scope in unique_dec} # now linking parents to their children # for parent, scope in zip(parent_layer.nodes(), # parent_scope_list): # dec_scopes = dec_scope_list[scope] # for dec in dec_scopes: # retrieving children # adding it # parent.add_child(children_dict[dec]) # we already have the nodes and their scopes # children_list = [child for child in children_dict.values()] # scope_list = [scope for scope in children_dict.keys()] sum_layer = SumLayerLinked(nodes=children_list) return sum_layer, scope_list
def test_spn_mpe_eval_and_traversal(): # create initial layer node1 = Node() node2 = Node() node3 = Node() node4 = Node() node5 = Node() input_layer = CategoricalInputLayer([node1, node2, node3, node4, node5]) # top layer made by 3 sum nodes sum1 = SumNode() sum2 = SumNode() sum3 = SumNode() # linking to input nodes weight11 = 0.3 sum1.add_child(node1, weight11) weight12 = 0.3 sum1.add_child(node2, weight12) weight13 = 0.4 sum1.add_child(node3, weight13) weight22 = 0.15 sum2.add_child(node2, weight22) weight23 = 0.15 sum2.add_child(node3, weight23) weight24 = 0.7 sum2.add_child(node4, weight24) weight33 = 0.4 sum3.add_child(node3, weight33) weight34 = 0.25 sum3.add_child(node4, weight34) weight35 = 0.35 sum3.add_child(node5, weight35) sum_layer = SumLayer([sum1, sum2, sum3]) # another layer with two product nodes prod1 = ProductNode() prod2 = ProductNode() prod1.add_child(sum1) prod1.add_child(sum2) prod2.add_child(sum2) prod2.add_child(sum3) prod_layer = ProductLayer([prod1, prod2]) # root layer, double sum root1 = SumNode() root2 = SumNode() weightr11 = 0.5 root1.add_child(prod1, weightr11) weightr12 = 0.5 root1.add_child(prod2, weightr12) weightr21 = 0.9 root2.add_child(prod1, weightr21) weightr22 = 0.1 root2.add_child(prod2, weightr22) root_layer = SumLayer([root1, root2]) # create the spn spn = Spn(input_layer=input_layer, layers=[sum_layer, prod_layer, root_layer]) print('===================') print(spn) print('===================') # setting the input values val1 = 0.0 node1.set_val(val1) val2 = 0.5 node2.set_val(val2) val3 = 0.3 node3.set_val(val3) val4 = 1.0 node4.set_val(val4) val5 = 0.0 node5.set_val(val5) # evaluating the spn with MPE inference res = spn.test_mpe_eval() print('spn eval\'d', res) # testing it # # testing the max layer max1 = max(val1 * weight11, val2 * weight12, val3 * weight13) max2 = max(val2 * weight22, val3 * weight23, val4 * weight24) max3 = max(val3 * weight33, val4 * weight34, val5 * weight35) log_max1 = log(max1) if not numpy.isclose(max1, 0) else LOG_ZERO log_max2 = log(max2) if not numpy.isclose(max2, 0) else LOG_ZERO log_max3 = log(max3) if not numpy.isclose(max3, 0) else LOG_ZERO print('expected max vals {0}, {1}, {2}'.format(log_max1, log_max2, log_max3)) print('found max vals {0}, {1}, {2}'.format(sum1.log_val, sum2.log_val, sum3.log_val)) if IS_LOG_ZERO(log_max1): assert IS_LOG_ZERO(sum1.log_val) else: assert_almost_equal(log_max1, sum1.log_val) if IS_LOG_ZERO(log_max2): assert IS_LOG_ZERO(sum2.log_val) else: assert_almost_equal(log_max2, sum2.log_val) if IS_LOG_ZERO(log_max3): assert IS_LOG_ZERO(sum3.log_val) else: assert_almost_equal(log_max3, sum3.log_val) # product layer is assumed to be fine, but let's check # it anyways prod_val1 = max1 * max2 prod_val2 = max2 * max3 prod_log_val1 = log_max1 + log_max2 prod_log_val2 = log_max2 + log_max3 print('exp prod vals {0}, {1}'.format(prod_log_val1, prod_log_val2)) print('rea prod vals {0}, {1}'.format(prod1.log_val, prod2.log_val)) if IS_LOG_ZERO(prod_log_val1): assert IS_LOG_ZERO(prod1.log_val) else: assert_almost_equal(prod_log_val1, prod1.log_val) if IS_LOG_ZERO(prod_log_val2): assert IS_LOG_ZERO(prod2.log_val) else: assert_almost_equal(prod_log_val2, prod2.log_val) # root layer, again a sum layer root_val1 = max(prod_val1 * weightr11, prod_val2 * weightr12) root_val2 = max(prod_val1 * weightr21, prod_val2 * weightr22) root_log_val1 = log(root_val1) if not numpy.isclose( root_val1, 0) else LOG_ZERO root_log_val2 = log(root_val2) if not numpy.isclose( root_val2, 0) else LOG_ZERO print('exp root vals {0}, {1}'.format(root_log_val1, root_log_val2)) print('found ro vals {0}, {1}'.format(root1.log_val, root2.log_val)) if IS_LOG_ZERO(root_log_val1): assert IS_LOG_ZERO(root1.log_val) else: assert_almost_equal(root_log_val1, root1.log_val) if IS_LOG_ZERO(root_log_val2): assert IS_LOG_ZERO(root2.log_val) else: assert_almost_equal(root_log_val2, root2.log_val) # now we are traversing top down the net print('mpe traversing') for i, j, k in spn.mpe_traversal(): print(i, j, k)
def fit_structure(self, data): # # a queue containing the data slices to process slices_to_process = deque() # a stack for building nodes building_stack = deque() # a dict to keep track of id->nodes node_id_assoc = {} # creating the first slice whole_slice = DataSlice.whole_slice(data.shape[0], data.shape[1]) slices_to_process.append(whole_slice) cluster_first = self._cluster_first # # iteratively process & split slices # while slices_to_process: # process a slice current_slice = slices_to_process.popleft() # pointers to the current data slice current_instances = current_slice.instance_ids current_features = current_slice.feature_ids current_id = current_slice.id n_features = len(current_features) # if n_features > 1: # # # print("removing Zeros") # datarowsIdx = numpy.sum(data[current_instances, :][:, current_features], 1) > 0 # if not any(datarowsIdx): # datarowsIdx[0] = True # current_instances = current_slice.instance_ids[datarowsIdx] n_instances = len(current_instances) # if n_instances == 0: # #too strong cutting the zeroes # current_instances = [current_slice.instance_ids[0]] # n_instances = len(current_instances) slice_data_rows = data[current_instances, :] current_slice_data = slice_data_rows[:, current_features] # is this a leaf node or we can split? if n_features == 1 and (current_slice.doNotCluster or n_instances <= self._min_instances_slice): (feature_id, ) = current_features if self.family == "poisson": leaf_node = PoissonNode(data, current_instances, current_features) elif self.family == "gaussian": leaf_node = GaussianNode(data, current_instances, current_features) # storing links # input_nodes.append(leaf_node) leaf_node.id = current_id node_id_assoc[current_id] = leaf_node # elif (current_slice_data.shape[0] < self._min_instances_slice): # elif ( (n_instances <= self._min_instances_slice and n_features > 1) and current_slice_data.shape[0] < self._min_instances_slice): # elif ((n_instances <= self._min_instances_slice and n_features > 1)): elif n_features > 1 and (current_slice.doNotCluster or n_instances <= self._min_instances_slice): # print('into naive factorization') child_slices = [ DataSlice(current_instances, [feature_id]) for feature_id in current_features ] slices_to_process.extend(child_slices) #children_ids = [child.id for child in child_slices] for child_slice in child_slices: child_slice.doNotCluster = current_slice.doNotCluster current_slice.add_child(child_slice) current_slice.type = ProductNode building_stack.append(current_slice) prod_node = ProductNode(data, current_instances, current_features) prod_node.id = current_id node_id_assoc[current_id] = prod_node else: split_on_features = False # first_run = False # # first run is a split on rows if n_features == 1 or cluster_first: cluster_first = False else: if self._ind_test_method == "pairwise_treeglm" or self._ind_test_method == "subsample": fcdata = current_slice_data if self._ind_test_method == "subsample": #sampled_rows = 2000 #sampled_rows = math.floor(current_slice_data.shape[0]*10/100) sampled_rows = self._sub_sample_rows if sampled_rows < current_slice_data.shape[0]: fcdata = current_slice_data[ numpy.random.choice( current_slice_data.shape[0], sampled_rows, replace=False)] else: fcdata = current_slice_data #Using R #from pdn.independenceptest import getIndependentGroups #feature_clusters = retrieve_clustering(getIndependentGroups(fcdata, alpha=self._alpha, family=self.family), current_features) feature_clusters = retrieve_clustering( getIndependentGroupsStabilityTest( fcdata, alpha=self._alpha), current_features) elif self._ind_test_method == "KMeans": feature_clusters = retrieve_clustering( cluster_rows( (data[current_instances, :][:, current_features] ).T, n_clusters=2, cluster_method=self._row_cluster_method, n_iters=self._n_iters, n_restarts=self._n_restarts, cluster_prep_method="sqrt", cluster_penalty=self._cluster_penalty, rand_gen=self._rand_gen, sklearn_args=self._sklearn_args), current_instances) split_on_features = len(feature_clusters) > 1 # # have dependent components been found? if split_on_features: # # splitting on columns # print('---> Splitting on features') # print(feature_clusters) slices = [ DataSlice(current_instances, cluster) for cluster in feature_clusters ] slices_to_process.extend(slices) current_slice.type = ProductNode building_stack.append(current_slice) for child_slice in slices: current_slice.add_child(child_slice) prod_node = ProductNode(data, current_instances, current_features) prod_node.id = current_id node_id_assoc[current_id] = prod_node else: # print('---> Splitting on rows') k_row_clusters = min(self._n_cluster_splits, n_instances - 1) if n_features == 1: # do one kmeans run with K large enough to split into N min instances k_row_clusters = math.floor( n_instances / self._min_instances_slice) + 1 k_row_clusters = min(k_row_clusters, n_instances - 1) clustering = retrieve_clustering( cluster_rows( data[current_instances, :][:, current_features], n_clusters=k_row_clusters, cluster_method=self._row_cluster_method, n_iters=self._n_iters, n_restarts=self._n_restarts, cluster_prep_method=self._cluster_prep_method, cluster_penalty=self._cluster_penalty, rand_gen=self._rand_gen, sklearn_args=self._sklearn_args), current_instances) cluster_slices = [ DataSlice(cluster, current_features) for cluster in clustering ] if len(clustering) < k_row_clusters: for cluster_slice in cluster_slices: cluster_slice.doNotCluster = True n_instances_clusters = sum( [len(cluster) for cluster in clustering]) cluster_weights = [ len(cluster) / n_instances_clusters for cluster in clustering ] slices_to_process.extend(cluster_slices) current_slice.type = SumNode building_stack.append(current_slice) for child_slice, child_weight in zip( cluster_slices, cluster_weights): current_slice.add_child(child_slice, child_weight) sum_node = SumNode(data, current_instances, current_features) sum_node.id = current_id node_id_assoc[current_id] = sum_node root_node = SpnFactory.pruned_spn_from_slices(node_id_assoc, building_stack, True) spn = SpnFactory.layered_linked_spn(root_node, data, self.config) return spn
def test_layered_pruned_linked_spn_cltree(): # # creating all the data slices # the slicing is a fake stub rows = 5 cols = 5 var = 1 values = 2 vars = [2, 3] var_values = [2, 2] s_data = numpy.array([[0, 1], [1, 1], [1, 0], [0, 0]]) node_1 = SumNode() node_1.id = 1 node_2 = ProductNode() node_2.id = 2 node_3 = SumNode() node_3.id = 3 # adding first level weight_12 = 0.4 weight_13 = 0.6 node_1.add_child(node_2, weight_12) node_1.add_child(node_3, weight_13) node_4 = ProductNode() node_4.id = 4 leaf_5 = CategoricalSmoothedNode(var, values) leaf_5.id = 5 # not adding the slice to the stack node_2.add_child(node_4) node_2.add_child(leaf_5) node_6 = SumNode() node_6.id = 6 node_7 = SumNode() node_7.id = 7 weight_36 = 0.1 weight_37 = 0.9 node_3.add_child(node_6, weight_36) node_3.add_child(node_7, weight_37) node_8 = ProductNode() node_8.id = 8 # # this is a cltree leaf_15 = CLTreeNode(vars=vars, var_values=var_values, data=s_data) leaf_15.id = 15 node_4.add_child(node_8) node_4.add_child(leaf_15) leaf_13 = CategoricalSmoothedNode(var, values) leaf_13.id = 13 leaf_14 = CLTreeNode(vars=vars, var_values=var_values, data=s_data) leaf_14.id = 14 node_8.add_child(leaf_13) node_8.add_child(leaf_14) leaf_9 = CLTreeNode(vars=vars, var_values=var_values, data=s_data) leaf_9.id = 9 node_10 = ProductNode() node_10.id = 10 leaf_18 = CategoricalSmoothedNode(var, values) leaf_18.id = 18 leaf_19 = CategoricalSmoothedNode(var, values) leaf_19.id = 19 node_10.add_child(leaf_18) node_10.add_child(leaf_19) weight_69 = 0.3 weight_610 = 0.7 node_6.add_child(leaf_9, weight_69) node_6.add_child(node_10, weight_610) node_11 = ProductNode() node_11.id = 11 leaf_20 = CategoricalSmoothedNode(var, values) leaf_20.id = 20 leaf_21 = CategoricalSmoothedNode(var, values) leaf_21.id = 21 node_11.add_child(leaf_20) node_11.add_child(leaf_21) node_12 = ProductNode() node_12.id = 12 leaf_22 = CLTreeNode(vars=vars, var_values=var_values, data=s_data) leaf_22.id = 22 leaf_23 = CategoricalSmoothedNode(var, values) leaf_23.id = 23 node_12.add_child(leaf_22) node_12.add_child(leaf_23) weight_711 = 0.5 weight_712 = 0.5 node_7.add_child(node_11, weight_711) node_7.add_child(node_12, weight_712) print('Added nodes') root_node = SpnFactory.layered_pruned_linked_spn(node_1) print('ROOT nODE', root_node) spn = SpnFactory.layered_linked_spn(root_node) print('SPN', spn) assert spn.n_layers() == 3 for i, layer in enumerate(spn.top_down_layers()): if i == 0: assert layer.n_nodes() == 1 elif i == 1: assert layer.n_nodes() == 4 elif i == 2: assert layer.n_nodes() == 10