def mnist_01(inputs, num_decomps, num_subsets, num_mixtures, num_input_mixtures, balanced, input_dist, node_type, inf_type, log=False): # Learning Parameters additive_smoothing = 100 min_additive_smoothing = 1 # Weight initialization weight_init_value = tf.initializers.random_uniform(10, 11) # Generate SPN structure dense_gen = spn.DenseSPNGenerator(num_decomps=num_decomps, num_subsets=num_subsets, num_mixtures=num_mixtures, input_dist=(spn.DenseSPNGenerator. InputDist.RAW if input_dist is "RAW" else spn. DenseSPNGenerator. InputDist.MIXTURE), num_input_mixtures=num_input_mixtures, balanced=balanced, node_type=node_type) root0 = dense_gen.generate(inputs, root_name="root_0") root1 = dense_gen.generate(inputs, root_name="root_1") root = spn.Sum(root0, root1, name="root") spn.generate_weights(root, initializer=weight_init_value) latent = root.generate_latent_indicators() # Add EM Learning additive_smoothing_var = tf.Variable(additive_smoothing, dtype=spn.conf.dtype) learning = spn.HardEMLearning(root, log=log, value_inference_type=inf_type, additive_smoothing=additive_smoothing_var) return root, latent, learning, additive_smoothing, min_additive_smoothing, \ additive_smoothing_var
def test_gradient_on_dense_spn(self, num_decomps, num_subsets, num_mixtures, input_dist, num_vars, num_components, softplus): batch_size = 9 mean_init = np.arange(num_vars * num_components).reshape( num_vars, num_components) gl = spn.GaussianLeaf(num_vars=num_vars, num_components=num_components, loc_init=mean_init, softplus_scale=softplus) gen = spn.DenseSPNGenerator( num_decomps=num_decomps, num_subsets=num_subsets, num_mixtures=num_mixtures, node_type=spn.DenseSPNGenerator.NodeType.LAYER, input_dist=input_dist) root = gen.generate(gl, root_name="root") with tf.name_scope("Weights"): spn.generate_weights(root, tf.initializers.random_uniform(0.0, 1.0), log=True) init = spn.initialize_weights(root) self.assertTrue(root.is_valid()) log_val = root.get_log_value() spn_grad = spn.Gradient(log=True) spn_grad.get_gradients(root) mean_grad_custom, var_grad_custom = gl._compute_gradient( spn_grad.gradients[gl]) mean_grad_tf, var_grad_tf = tf.gradients( log_val, [gl.loc_variable, gl.scale_variable]) fd = {gl: np.random.rand(batch_size, num_vars)} with self.test_session() as sess: sess.run(init) mu_grad_custom_val, var_grad_custom_val = sess.run( [mean_grad_custom, var_grad_custom], fd) mu_grad_tf_val, var_grad_tf_val = sess.run( [mean_grad_tf, var_grad_tf], fd) self.assertAllClose(mu_grad_custom_val, mu_grad_tf_val, atol=1e-4, rtol=1e-4) self.assertAllClose(var_grad_custom_val, var_grad_tf_val, atol=1e-4, rtol=1e-4)
def mnist_all(inputs, num_decomps, num_subsets, num_mixtures, num_input_mixtures, balanced, input_dist, node_type, inf_type, log=False): # Learning Parameters additive_smoothing = 0 min_additive_smoothing = 0 initial_accum_value = 20 # Weight initialization weight_init_value = tf.initializers.random_uniform(0, 1) # Add random values before max add_random = None use_unweighted = True # Generate SPN structure dense_gen = spn.DenseSPNGenerator( num_decomps=num_decomps, num_subsets=num_subsets, num_mixtures=num_mixtures, input_dist=(spn.DenseSPNGenerator.InputDist.RAW if input_dist is "RAW" else spn.DenseSPNGenerator.InputDist.MIXTURE), num_input_mixtures=num_input_mixtures, balanced=balanced, node_type=node_type) class_roots = [ dense_gen.generate(inputs, root_name=("Class_%d" % i)) for i in range(10) ] root = spn.Sum(*class_roots, name="root") spn.generate_weights(root, init_value=weight_init_value) latent = root.generate_ivs() # Add EM Learning additive_smoothing_var = tf.Variable(additive_smoothing, dtype=spn.conf.dtype) learning = spn.EMLearning(root, log=log, value_inference_type=inf_type, additive_smoothing=additive_smoothing_var, add_random=add_random, initial_accum_value=initial_accum_value, use_unweighted=use_unweighted) return root, latent, learning, additive_smoothing, min_additive_smoothing, \ additive_smoothing_var
def generic_dense_test(self, name, num_decomps, num_subsets, num_mixtures, input_dist, num_input_mixtures): """A generic test for DenseSPNGenerator.""" v1 = spn.IVs(num_vars=3, num_vals=2, name="IVs1") v2 = spn.IVs(num_vars=3, num_vals=2, name="IVs2") gen = spn.DenseSPNGenerator(num_decomps=num_decomps, num_subsets=num_subsets, num_mixtures=num_mixtures, input_dist=input_dist, num_input_mixtures=num_input_mixtures) # Generating SPN root = gen.generate(v1, v2) # Generating random weights with tf.name_scope("Weights"): spn.generate_weights(root, tf.initializers.random_uniform(0.0, 1.0)) # Generating weight initializers init = spn.initialize_weights(root) # Testing validity self.assertTrue(root.is_valid()) # Generating value ops v = root.get_value() v_log = root.get_log_value() # Creating session with self.test_session() as sess: # Initializing weights init.run() # Computing all values feed = np.array(list(itertools.product(range(2), repeat=6))) feed_v1 = feed[:, :3] feed_v2 = feed[:, 3:] out = sess.run(v, feed_dict={v1: feed_v1, v2: feed_v2}) out_log = sess.run(tf.exp(v_log), feed_dict={ v1: feed_v1, v2: feed_v2 }) # Test if partition function is 1.0 self.assertAlmostEqual(out.sum(), 1.0, places=6) self.assertAlmostEqual(out_log.sum(), 1.0, places=6) self.write_tf_graph(sess, self.sid(), self.cid())
def test_generte_set_errors(self): """Detecting structure errors in __generate_set""" gen = spn.DenseSPNGenerator(num_decomps=2, num_subsets=3, num_mixtures=2) v1 = spn.IVs(num_vars=2, num_vals=4) v2 = spn.ContVars(num_vars=3, name="ContVars1") v3 = spn.ContVars(num_vars=2, name="ContVars2") s1 = spn.Sum(v3, v2) n1 = spn.Concat(v2) with self.assertRaises(spn.StructureError): gen._DenseSPNGenerator__generate_set([ spn.Input(v1, [0, 3, 2, 6, 7]), spn.Input(v2, [1, 2]), spn.Input(s1, None), spn.Input(n1, None) ])
def dense_block(inputs, num_decomps, num_subsets, num_mixtures, num_input_mixtures, balanced, input_dist, inf_type, log=False): # Set node-type as single-node node_type = spn.DenseSPNGenerator.NodeType.BLOCK # Create a dense generator gen = spn.DenseSPNGenerator( num_decomps=num_decomps, num_subsets=num_subsets, num_mixtures=num_mixtures, num_input_mixtures=num_input_mixtures, balanced=balanced, node_type=node_type, input_dist=(spn.DenseSPNGenerator.InputDist.RAW if input_dist is "RAW" else spn.DenseSPNGenerator.InputDist.MIXTURE)) # Generate a dense SPN, with single-op nodes, and all weights in the network root = gen.generate(inputs, root_name="root") spn.generate_weights(root, tf.initializers.random_uniform(0.0, 1.0)) # Generate path ops based on inf_type and log if log: mpe_path_gen = spn.MPEPath(value_inference_type=inf_type, log=True) else: mpe_path_gen = spn.MPEPath(value_inference_type=inf_type, log=False) mpe_path_gen.get_mpe_path(root) path_ops = [ mpe_path_gen.counts[inp] for inp in (inputs if isinstance(inputs, list) else [inputs]) ] return root, spn.initialize_weights(root), path_ops
def test_traversing_on_dense(self): """Compare traversal algs on dense SPN""" def fun1(node, *args): counter[0] += 1 def fun2(node, *args): counter[0] += 1 if node.is_op: return [None] * len(node.inputs) # Generate dense graph v1 = spn.IndicatorLeaf(num_vars=3, num_vals=2, name="IndicatorLeaf1") v2 = spn.IndicatorLeaf(num_vars=3, num_vals=2, name="IndicatorLeaf2") gen = spn.DenseSPNGenerator(num_decomps=2, num_subsets=3, num_mixtures=2, input_dist=spn.DenseSPNGenerator.InputDist.MIXTURE, num_input_mixtures=None) root = gen.generate(v1, v2) spn.generate_weights(root) # Run traversal algs and count nodes counter = [0] spn.compute_graph_up_down(root, down_fun=fun2, graph_input=1) c1 = counter[0] counter = [0] spn.compute_graph_up(root, val_fun=fun1) c2 = counter[0] counter = [0] spn.traverse_graph(root, fun=fun1, skip_params=False) c3 = counter[0] # Compare self.assertEqual(c1, c3) self.assertEqual(c2, c3)
def test_generte_set(self): """Generation of sets of inputs with __generate_set""" gen = spn.DenseSPNGenerator(num_decomps=2, num_subsets=3, num_mixtures=2) v1 = spn.IVs(num_vars=2, num_vals=4) v2 = spn.ContVars(num_vars=3, name="ContVars1") v3 = spn.ContVars(num_vars=2, name="ContVars2") s1 = spn.Sum(v3) n1 = spn.Concat(v2) out = gen._DenseSPNGenerator__generate_set([ spn.Input(v1, [0, 3, 2, 6, 7]), spn.Input(v2, [1, 2]), spn.Input(s1, None), spn.Input(n1, None) ]) # scope_dict: # Scope({IVs(0x7f00cb4049b0):0}): {(IVs(0x7f00cb4049b0), 0), # (IVs(0x7f00cb4049b0), 2), # (IVs(0x7f00cb4049b0), 3)}, # Scope({IVs(0x7f00cb4049b0):1}): {(IVs(0x7f00cb4049b0), 7), # (IVs(0x7f00cb4049b0), 6)}, # Scope({ContVars1(0x7f00b7982ef0):1}): {(Concat(0x7f00cb404d68), 1), # (ContVars1(0x7f00b7982ef0), 1)}, # Scope({ContVars1(0x7f00b7982ef0):2}): {(Concat(0x7f00cb404d68), 2), # (ContVars1(0x7f00b7982ef0), 2)}, # Scope({ContVars1(0x7f00b7982ef0):0}): {(Concat(0x7f00cb404d68), 0)}, # Scope({ContVars2(0x7f00cb391eb8):0, ContVars2(0x7f00cb391eb8):1}): { # (Sum(0x7f00cb404a90), 0)}} # Since order is undetermined, we check items self.assertEqual(len(out), 6) self.assertIn(tuple(sorted([(v2, 1), (n1, 1)])), out) self.assertIn(tuple(sorted([(v2, 2), (n1, 2)])), out) self.assertIn(tuple(sorted([(n1, 0)])), out) self.assertIn(tuple(sorted([(v1, 0), (v1, 2), (v1, 3)])), out) self.assertIn(tuple(sorted([(v1, 6), (v1, 7)])), out) self.assertIn(tuple(sorted([(s1, 0)])), out)
def dense_block(inputs, num_decomps, num_subsets, num_mixtures, num_input_mixtures, balanced, input_dist, inf_type, log=False): # Set node-type as single-node node_type = spn.DenseSPNGenerator.NodeType.BLOCK # Create a dense generator gen = spn.DenseSPNGenerator( num_decomps=num_decomps, num_subsets=num_subsets, num_mixtures=num_mixtures, num_input_mixtures=num_input_mixtures, balanced=balanced, node_type=node_type, input_dist=(spn.DenseSPNGenerator.InputDist.RAW if input_dist is "RAW" else spn.DenseSPNGenerator.InputDist.MIXTURE)) # Generate a dense SPN, with block-nodes, and all weights in the network root = gen.generate(inputs, root_name="root") spn.generate_weights(root, tf.initializers.random_uniform(0.0, 1.0)) # Generate value ops based on inf_type and log if log: value_op = root.get_log_value(inference_type=inf_type) else: value_op = root.get_value(inference_type=inf_type) return root, spn.initialize_weights(root), value_op
def test_generte_set(self): """Generation of sets of inputs with __generate_set""" gen = spn.DenseSPNGenerator(num_decomps=2, num_subsets=3, num_mixtures=2) v1 = spn.IVs(num_vars=2, num_vals=4) v2 = spn.ContVars(num_vars=3, name="ContVars1") v3 = spn.ContVars(num_vars=2, name="ContVars2") s1 = spn.Sum(v3) n1 = spn.Concat(v2) out = gen._DenseSPNGenerator__generate_set([ spn.Input(v1, [0, 3, 2, 6, 7]), spn.Input(v2, [1, 2]), spn.Input(s1, None), spn.Input(n1, None) ]) # Since order is undetermined, we check items self.assertEqual(len(out), 6) self.assertIn(tuple(sorted([(v2, 1), (n1, 1)])), out) self.assertIn(tuple(sorted([(v2, 2), (n1, 2)])), out) self.assertIn(tuple(sorted([(n1, 0)])), out) self.assertIn(tuple(sorted([(v1, 0), (v1, 2), (v1, 3)])), out) self.assertIn(tuple(sorted([(v1, 6), (v1, 7)])), out) self.assertIn(tuple(sorted([(s1, 0)])), out)
def test_generate_spn(self, num_decomps, num_subsets, num_mixtures, num_input_mixtures, input_dims, input_dist, balanced, node_type, log_weights): """A generic test for DenseSPNGenerator.""" if input_dist == spn.DenseSPNGenerator.InputDist.RAW \ and num_input_mixtures != 1: # Redundant test case, so just return return # Input parameters num_inputs = input_dims[0] num_vars = input_dims[1] num_vals = 2 printc("\n- num_inputs: %s" % num_inputs) printc("- num_vars: %s" % num_vars) printc("- num_vals: %s" % num_vals) printc("- num_decomps: %s" % num_decomps) printc("- num_subsets: %s" % num_subsets) printc("- num_mixtures: %s" % num_mixtures) printc("- input_dist: %s" % ("MIXTURE" if input_dist == spn.DenseSPNGenerator.InputDist.MIXTURE else "RAW")) printc("- balanced: %s" % balanced) printc("- num_input_mixtures: %s" % num_input_mixtures) printc("- node_type: %s" % ("SINGLE" if node_type == spn.DenseSPNGenerator.NodeType.SINGLE else "BLOCK" if node_type == spn.DenseSPNGenerator.NodeType.BLOCK else "LAYER")) printc("- log_weights: %s" % log_weights) # Inputs inputs = [ spn.IVs(num_vars=num_vars, num_vals=num_vals, name=("IVs_%d" % (i + 1))) for i in range(num_inputs) ] gen = spn.DenseSPNGenerator(num_decomps=num_decomps, num_subsets=num_subsets, num_mixtures=num_mixtures, input_dist=input_dist, balanced=balanced, num_input_mixtures=num_input_mixtures, node_type=node_type) # Generate Sub-SPNs sub_spns = [ gen.generate(*inputs, root_name=("sub_root_%d" % (i + 1))) for i in range(3) ] # Generate random weights for the first sub-SPN with tf.name_scope("Weights"): spn.generate_weights(sub_spns[0], tf.initializers.random_uniform(0.0, 1.0), log=log_weights) # Initialize weights of the first sub-SPN sub_spn_init = spn.initialize_weights(sub_spns[0]) # Testing validity of the first sub-SPN self.assertTrue(sub_spns[0].is_valid()) # Generate value ops of the first sub-SPN sub_spn_v = sub_spns[0].get_value() sub_spn_v_log = sub_spns[0].get_log_value() # Generate path ops of the first sub-SPN sub_spn_mpe_path_gen = spn.MPEPath(log=False) sub_spn_mpe_path_gen_log = spn.MPEPath(log=True) sub_spn_mpe_path_gen.get_mpe_path(sub_spns[0]) sub_spn_mpe_path_gen_log.get_mpe_path(sub_spns[0]) sub_spn_path = [sub_spn_mpe_path_gen.counts[inp] for inp in inputs] sub_spn_path_log = [ sub_spn_mpe_path_gen_log.counts[inp] for inp in inputs ] # Collect all weight nodes of the first sub-SPN sub_spn_weight_nodes = [] def fun(node): if node.is_param: sub_spn_weight_nodes.append(node) spn.traverse_graph(sub_spns[0], fun=fun) # Generate an upper-SPN over sub-SPNs products_lower = [] for sub_spn in sub_spns: products_lower.append([v.node for v in sub_spn.values]) num_top_mixtures = [2, 1, 3] sums_lower = [] for prods, num_top_mix in zip(products_lower, num_top_mixtures): if node_type == spn.DenseSPNGenerator.NodeType.SINGLE: sums_lower.append( [spn.Sum(*prods) for _ in range(num_top_mix)]) elif node_type == spn.DenseSPNGenerator.NodeType.BLOCK: sums_lower.append([spn.ParSums(*prods, num_sums=num_top_mix)]) else: sums_lower.append([ spn.SumsLayer(*prods * num_top_mix, num_or_size_sums=num_top_mix) ]) # Generate upper-SPN root = gen.generate(*list(itertools.chain(*sums_lower)), root_name="root") # Generate random weights for the SPN with tf.name_scope("Weights"): spn.generate_weights(root, tf.initializers.random_uniform(0.0, 1.0), log=log_weights) # Initialize weight of the SPN spn_init = spn.initialize_weights(root) # Testing validity of the SPN self.assertTrue(root.is_valid()) # Generate value ops of the SPN spn_v = root.get_value() spn_v_log = root.get_log_value() # Generate path ops of the SPN spn_mpe_path_gen = spn.MPEPath(log=False) spn_mpe_path_gen_log = spn.MPEPath(log=True) spn_mpe_path_gen.get_mpe_path(root) spn_mpe_path_gen_log.get_mpe_path(root) spn_path = [spn_mpe_path_gen.counts[inp] for inp in inputs] spn_path_log = [spn_mpe_path_gen_log.counts[inp] for inp in inputs] # Collect all weight nodes in the SPN spn_weight_nodes = [] def fun(node): if node.is_param: spn_weight_nodes.append(node) spn.traverse_graph(root, fun=fun) # Create a session with self.test_session() as sess: # Initializing weights sess.run(sub_spn_init) sess.run(spn_init) # Generate input feed feed = np.array( list( itertools.product(range(num_vals), repeat=(num_inputs * num_vars)))) batch_size = feed.shape[0] feed_dict = {} for inp, f in zip(inputs, np.split(feed, num_inputs, axis=1)): feed_dict[inp] = f # Compute all values and paths of sub-SPN sub_spn_out = sess.run(sub_spn_v, feed_dict=feed_dict) sub_spn_out_log = sess.run(tf.exp(sub_spn_v_log), feed_dict=feed_dict) sub_spn_out_path = sess.run(sub_spn_path, feed_dict=feed_dict) sub_spn_out_path_log = sess.run(sub_spn_path_log, feed_dict=feed_dict) # Compute all values and paths of the complete SPN spn_out = sess.run(spn_v, feed_dict=feed_dict) spn_out_log = sess.run(tf.exp(spn_v_log), feed_dict=feed_dict) spn_out_path = sess.run(spn_path, feed_dict=feed_dict) spn_out_path_log = sess.run(spn_path_log, feed_dict=feed_dict) # Test if partition function of the sub-SPN and of the # complete SPN is 1.0 self.assertAlmostEqual(sub_spn_out.sum(), 1.0, places=6) self.assertAlmostEqual(sub_spn_out_log.sum(), 1.0, places=6) self.assertAlmostEqual(spn_out.sum(), 1.0, places=6) self.assertAlmostEqual(spn_out_log.sum(), 1.0, places=6) # Test if the sum of counts for each value of each variable # (6 variables, with 2 values each) = batch-size / num-vals self.assertEqual( np.sum(np.hstack(sub_spn_out_path), axis=0).tolist(), [batch_size // num_vals] * num_inputs * num_vars * num_vals) self.assertEqual( np.sum(np.hstack(sub_spn_out_path_log), axis=0).tolist(), [batch_size // num_vals] * num_inputs * num_vars * num_vals) self.assertEqual( np.sum(np.hstack(spn_out_path), axis=0).tolist(), [batch_size // num_vals] * num_inputs * num_vars * num_vals) self.assertEqual( np.sum(np.hstack(spn_out_path_log), axis=0).tolist(), [batch_size // num_vals] * num_inputs * num_vars * num_vals)