def test_optimization(self):
np.random.seed(17)
d1 = np.random.normal(10, 5, size=2000).tolist()
d2 = np.random.normal(30, 5, size=2000).tolist()
data = d1 + d2
data = np.array(data).reshape((-1, 10))
data = data.astype(np.float32)
ds_context = Context(meta_types=[MetaType.REAL] * data.shape[1],
parametric_types=[Gaussian] * data.shape[1])
spn = learn_parametric(data, ds_context)
spn.weights = [0.8, 0.2]
spn.children[0].children[0].mean = 3.0
py_ll = np.sum(log_likelihood(spn, data))
print(spn.weights, spn.children[0].children[0].mean)
EM_optimization(spn, data, iterations=10)
print(spn.weights, spn.children[0].children[0].mean)
py_ll_opt = np.sum(log_likelihood(spn, data))
self.assertLessEqual(py_ll, py_ll_opt)
self.assertAlmostEqual(spn.weights[0], 0.5, 6)
self.assertAlmostEqual(spn.weights[1], 0.5, 6)
self.assertAlmostEqual(spn.children[0].children[0].mean, 10.50531, 4)
def test_sum_one_dimension(self):
add_node_likelihood(Leaf, identity_ll)
# test that we get basic computations right
spn = 0.5 * Leaf(scope=0) + 0.5 * Leaf(scope=0)
data = np.random.rand(10, 1)
self.assert_correct(spn, data, data)
spn = 0.1 * Leaf(scope=0) + 0.9 * Leaf(scope=0)
data = np.random.rand(10, 1)
self.assert_correct(spn, data, data)
# test that we can pass whatever dataset, and the scopes are being respected
# this is important for inner nodes
spn = 0.1 * Leaf(scope=0) + 0.9 * Leaf(scope=0)
data = np.random.rand(10, 3)
r = 0.1 * data[:, 0] + 0.9 * data[:, 0]
r = r.reshape(-1, 1)
self.assert_correct(spn, data, r)
# test that it fails if the weights are not normalized
spn = 0.1 * Leaf(scope=0) + 0.9 * Leaf(scope=0)
spn.weights[1] = 0.2
data = np.random.rand(10, 3)
with self.assertRaises(AssertionError):
l = likelihood(spn, data)
with self.assertRaises(AssertionError):
log_likelihood(spn, data)
# test the log space
spn = 0.1 * Leaf(scope=0) + 0.9 * Leaf(scope=0)
data = np.random.rand(10, 3)
r = 0.1 * data[:, 0] + 0.9 * data[:, 0]
r = r.reshape(-1, 1)
self.assert_correct(spn, data, r)
def EM_optimization(spn, data, iterations=5, node_updates=_node_updates, skip_validation=False, **kwargs):
if not skip_validation:
valid, err = is_valid(spn)
assert valid, "invalid spn: " + err
lls_per_node = np.zeros((data.shape[0], get_number_of_nodes(spn)))
for _ in range(iterations):
# one pass bottom up evaluating the likelihoods
log_likelihood(spn, data, dtype=data.dtype, lls_matrix=lls_per_node)
gradients = gradient_backward(spn, lls_per_node)
R = lls_per_node[:, 0]
for node_type, func in node_updates.items():
for node in get_nodes_by_type(spn, node_type):
func(
node,
node_lls=lls_per_node[:, node.id],
node_gradients=gradients[:, node.id],
root_lls=R,
all_lls=lls_per_node,
all_gradients=gradients,
data=data,
**kwargs
)
def mpe(
node,
input_data,
node_top_down_mpe=_node_top_down_mpe,
node_bottom_up_mpe_log=_node_bottom_up_mpe_log,
in_place=False,
):
valid, err = is_valid(node)
assert valid, err
assert np.all(
np.any(np.isnan(input_data), axis=1)
), "each row must have at least a nan value where the samples will be substituted"
if in_place:
data = input_data
else:
data = np.array(input_data)
nodes = get_nodes_by_type(node)
lls_per_node = np.zeros((data.shape[0], len(nodes)))
# one pass bottom up evaluating the likelihoods
log_likelihood(node, data, dtype=data.dtype, node_log_likelihood=node_bottom_up_mpe_log, lls_matrix=lls_per_node)
instance_ids = np.arange(data.shape[0])
# one pass top down to decide on the max branch until it reaches a leaf, then it fills the nan slot with the mode
eval_spn_top_down(node, node_top_down_mpe, parent_result=instance_ids, data=data, lls_per_node=lls_per_node)
return data
def EM_optimization(spn,
data,
iterations=5,
node_updates=_node_updates,
**kwargs):
for _ in range(iterations):
lls_per_node = np.zeros((data.shape[0], get_number_of_nodes(spn)))
# one pass bottom up evaluating the likelihoods
log_likelihood(spn, data, dtype=data.dtype, lls_matrix=lls_per_node)
gradients = gradient_backward(spn, lls_per_node)
R = lls_per_node[:, 0]
for node_type, func in node_updates.items(): # TODO: do in parallel
for node in get_nodes_by_type(spn, node_type):
func(node,
node_lls=lls_per_node[:, node.id],
node_gradients=gradients[:, node.id],
root_lls=R,
all_lls=lls_per_node,
all_gradients=gradients,
data=data,
**kwargs)
def calc_ll(wspn, data_train, data_pos, data_neg):
# calculate LL
log_msg = 'Log-likelihood calculating...'
print(log_msg)
logger.info(log_msg)
ll_train = log_likelihood(wspn, data_train)
ll_pos = log_likelihood(wspn, data_pos)
ll_neg = log_likelihood(wspn, data_neg)
log_msg = '---------median-----------'
print(log_msg)
logger.info(log_msg)
log_msg = 'LL_train=' + str(np.median(ll_train))
print(log_msg)
logger.info(log_msg)
log_msg = 'LL_test=' + str(np.median(ll_pos))
print(log_msg)
logger.info(log_msg)
log_msg = 'LL_ood=' + str(np.median(ll_neg))
print(log_msg)
logger.info(log_msg)
log_msg = '--------- mean -----------'
print(log_msg)
logger.info(log_msg)
log_msg = 'LL_train=' + str(np.mean(ll_train))
print(log_msg)
logger.info(log_msg)
log_msg = 'LL_test=' + str(np.mean(ll_pos))
print(log_msg)
logger.info(log_msg)
log_msg = 'LL_ood=' + str(np.mean(ll_neg))
print(log_msg)
logger.info(log_msg)
return ll_train, ll_pos, ll_neg
def sample_instances(node, input_data, rand_gen, node_sampling=_node_sampling, in_place=False):
"""
Implementing hierarchical sampling
"""
# first, we do a bottom-up pass to compute the likelihood taking into account marginals.
# then we do a top-down pass, to sample taking into account the likelihoods.
if in_place:
data = input_data
else:
data = np.array(input_data)
valid, err = is_valid(node)
assert valid, err
assert np.all(
np.any(np.isnan(data), axis=1)), "each row must have at least a nan value where the samples will be substituted"
nodes = get_nodes_by_type(node)
lls_per_node = np.zeros((data.shape[0], len(nodes)))
log_likelihood(node, data, dtype=data.dtype, lls_matrix=lls_per_node)
instance_ids = np.arange(data.shape[0])
eval_spn_top_down(node, node_sampling, input_vals=instance_ids, data=data, lls_per_node=lls_per_node,
rand_gen=rand_gen)
return data
def feature_gradient(node,
data,
node_gradient_functions=_node_feature_gradients,
lls_per_node=None):
"""
Feature gradients are computed for the input query and each feature using
the backwards automatic differentiation. In mathematicl terms, it computes the
partial derivatives \partial P(X) / \partial X_i
 
:param node: Node for the gradient calculation
:param data: data for the computation. NaN values are implicitely marginalized out
:param lls_per_node: optional for storing the intermediate results
"""
all_leaves = get_nodes_by_type(node, Leaf)
if not lls_per_node:
lls_per_node = np.full((data.shape[0], get_number_of_nodes(node)),
np.nan)
log_likelihood(node, data, lls_matrix=lls_per_node)
gradients = np.exp(gradient_backward(node, lls_per_node))
node_gradients = []
for spn_node in all_leaves:
i = spn_node.id
result = node_gradient_functions[type(spn_node)](spn_node, data)
node_gradients.append(result * gradients[:, i].reshape(-1, 1))
node_gradients = np.array(node_gradients)
return np.nansum(node_gradients, axis=0)
def test_Histogram_discrete_inference(self):
data = np.array([1, 1, 2, 3, 3, 3]).reshape(-1, 1)
ds_context = Context([MetaType.DISCRETE])
ds_context.add_domains(data)
hist = create_histogram_leaf(data, ds_context, [0], alpha=False)
prob = np.exp(log_likelihood(hist, data))
self.assertAlmostEqual(float(prob[0]), 2 / 6)
self.assertAlmostEqual(float(prob[1]), 2 / 6)
self.assertAlmostEqual(float(prob[2]), 1 / 6)
self.assertAlmostEqual(float(prob[3]), 3 / 6)
self.assertAlmostEqual(float(prob[4]), 3 / 6)
self.assertAlmostEqual(float(prob[5]), 3 / 6)
data = np.array([1, 1, 2, 3, 3, 3]).reshape(-1, 1)
ds_context = Context([MetaType.DISCRETE])
ds_context.add_domains(data)
hist = create_histogram_leaf(data, ds_context, [0], alpha=True)
# print(np.var(data.shape[0]))
prob = np.exp(log_likelihood(hist, data))
self.assertAlmostEqual(float(prob[0]), 3 / 9)
self.assertAlmostEqual(float(prob[1]), 3 / 9)
self.assertAlmostEqual(float(prob[2]), 2 / 9)
self.assertAlmostEqual(float(prob[3]), 4 / 9)
self.assertAlmostEqual(float(prob[4]), 4 / 9)
self.assertAlmostEqual(float(prob[5]), 4 / 9)
def test_ll_matrix(self):
add_node_likelihood(Leaf, sum_and_multiplier_ll)
node_1_1_1_1 = leaf(2, 1)
node_1_1_1_2 = leaf(2, 2)
node_1_1_1 = 0.7 * node_1_1_1_1 + 0.3 * node_1_1_1_2
node_1_1_2 = leaf([0, 1], 3)
node_1_1 = node_1_1_1 * node_1_1_2
node_1_2_1_1_1 = leaf(0, 5)
node_1_2_1_1_2 = leaf(1, 4)
node_1_2_1_1 = node_1_2_1_1_1 * node_1_2_1_1_2
node_1_2_1_2 = leaf([0, 1], 6)
node_1_2_1 = 0.1 * node_1_2_1_1 + 0.9 * node_1_2_1_2
node_1_2_2 = leaf(2, 3)
node_1_2 = node_1_2_1 * node_1_2_2
spn = 0.4 * node_1_1 + 0.6 * node_1_2
assign_ids(spn)
max_id = max([n.id for n in get_nodes_by_type(spn)])
data = np.random.rand(10, 10)
node_1_1_1_1_r = data[:, 2] * 1
node_1_1_1_2_r = data[:, 2] * 2
node_1_1_1_r = 0.7 * node_1_1_1_1_r + 0.3 * node_1_1_1_2_r
node_1_1_2_r = 3 * (data[:, 0] + data[:, 1])
node_1_1_r = node_1_1_1_r * node_1_1_2_r
node_1_2_1_1_1_r = data[:, 0] * 5
node_1_2_1_1_2_r = data[:, 1] * 4
node_1_2_1_1_r = node_1_2_1_1_1_r * node_1_2_1_1_2_r
node_1_2_1_2_r = 6 * (data[:, 0] + data[:, 1])
node_1_2_1_r = 0.1 * node_1_2_1_1_r + 0.9 * node_1_2_1_2_r
node_1_2_2_r = data[:, 2] * 3
node_1_2_r = node_1_2_1_r * node_1_2_2_r
spn_r = 0.4 * node_1_1_r + 0.6 * node_1_2_r
self.assert_correct(spn, data, spn_r)
lls = np.zeros((data.shape[0], max_id + 1))
likelihood(spn, data, lls_matrix=lls)
llls = np.zeros((data.shape[0], max_id + 1))
log_likelihood(spn, data, lls_matrix=llls)
self.assertTrue(np.alltrue(np.isclose(lls, np.exp(llls))))
self.assertTrue(np.alltrue(np.isclose(spn_r, lls[:, spn.id])))
self.assertTrue(np.alltrue(np.isclose(node_1_2_r, lls[:, node_1_2.id])))
self.assertTrue(np.alltrue(np.isclose(node_1_2_2_r, lls[:, node_1_2_2.id])))
self.assertTrue(np.alltrue(np.isclose(node_1_2_1_r, lls[:, node_1_2_1.id])))
self.assertTrue(np.alltrue(np.isclose(node_1_2_1_2_r, lls[:, node_1_2_1_2.id])))
self.assertTrue(np.alltrue(np.isclose(node_1_2_1_1_r, lls[:, node_1_2_1_1.id])))
self.assertTrue(np.alltrue(np.isclose(node_1_2_1_1_2_r, lls[:, node_1_2_1_1_2.id])))
self.assertTrue(np.alltrue(np.isclose(node_1_2_1_1_1_r, lls[:, node_1_2_1_1_1.id])))
self.assertTrue(np.alltrue(np.isclose(node_1_1_r, lls[:, node_1_1.id])))
self.assertTrue(np.alltrue(np.isclose(node_1_1_2_r, lls[:, node_1_1_2.id])))
self.assertTrue(np.alltrue(np.isclose(node_1_1_1_r, lls[:, node_1_1_1.id])))
self.assertTrue(np.alltrue(np.isclose(node_1_1_1_2_r, lls[:, node_1_1_1_2.id])))
self.assertTrue(np.alltrue(np.isclose(node_1_1_1_1_r, lls[:, node_1_1_1_1.id])))
def assert_correct(self, spn, data, result):
l = likelihood(spn, data)
self.assertEqual(l.shape[0], data.shape[0])
self.assertEqual(l.shape[1], 1)
self.assertTrue(np.alltrue(np.isclose(result.reshape(-1, 1), l)))
self.assertTrue(np.alltrue(np.isclose(np.log(l), log_likelihood(spn, data))))
self.assertTrue(np.alltrue(np.isclose(np.log(l), log_likelihood(spn, data, debug=True))))
self.assertTrue(np.alltrue(np.isclose(l, likelihood(spn, data, debug=True))))
def learn_CNET():
import numpy as np
np.random.seed(123)
train_data = np.random.binomial(1, [0.1, 0.2, 0.3, 0.4], size=(1000, 4))
print(np.mean(train_data, axis=0))
from spn.structure.leaves.cltree.CLTree import create_cltree_leaf
from spn.structure.Base import Context
from spn.structure.leaves.parametric.Parametric import Bernoulli
ds_context = Context(
parametric_types=[Bernoulli, Bernoulli, Bernoulli, Bernoulli
]).add_domains(train_data)
from spn.algorithms.LearningWrappers import learn_parametric, learn_cnet
cnet_naive_mle = learn_cnet(train_data,
ds_context,
cond="naive_mle",
min_instances_slice=20,
min_features_slice=1)
cnet_random = learn_cnet(train_data,
ds_context,
cond="random",
min_instances_slice=20,
min_features_slice=1)
from spn.algorithms.Statistics import get_structure_stats
from spn.io.Text import spn_to_str_equation
from spn.algorithms.Inference import log_likelihood
print(get_structure_stats(cnet_naive_mle))
print(spn_to_str_equation(cnet_naive_mle))
ll = log_likelihood(cnet_naive_mle, train_data)
print(np.mean(ll))
print(get_structure_stats(cnet_random))
print(spn_to_str_equation(cnet_random))
ll = log_likelihood(cnet_random, train_data)
print(np.mean(ll))
from spn.algorithms.MPE import mpe
train_data_mpe = train_data.astype(float)
train_data_mpe[:, 0] = np.nan
print(mpe(cnet_random, train_data_mpe)[:30])
ll = log_likelihood(cnet_random, train_data_mpe)
print(np.mean(ll))
def sample_induced_trees(node, data, rand_gen):
# this requires ids to be set, and to be ordered from 0 to N
validate_ids(node)
max_id = reset_node_counters(node)
lls = np.zeros((data.shape[0], max_id + 1))
log_likelihood(node, data, llls_matrix=lls)
map_rows_cols_to_node_id = np.zeros(data.shape, dtype=np.int64) - 1
# We do not collect all the Zs from all sum nodes as before, but only to those
# traversed during the top-down descent
def _sample_induced_trees(node, row_ids):
if len(row_ids) == 0:
return
node.row_ids = row_ids
if isinstance(node, Product):
for c in node.children:
_sample_induced_trees(c, row_ids)
return
if isinstance(node, Sum):
w_children_log_probs = np.zeros((len(row_ids), len(node.weights)))
for i, c in enumerate(node.children):
w_children_log_probs[:, i] = lls[row_ids, c.id] + np.log(
node.weights[i])
z_gumbels = rand_gen.gumbel(
loc=0,
scale=1,
# size=(w_children_log_probs.shape[1], w_children_log_probs.shape[0]))
size=(w_children_log_probs.shape[0],
w_children_log_probs.shape[1]))
g_children_log_probs = w_children_log_probs + z_gumbels
rand_child_branches = np.argmax(g_children_log_probs, axis=1)
for i, c in enumerate(node.children):
new_row_ids = row_ids[rand_child_branches == i]
node.edge_counts[i] = len(new_row_ids)
_sample_induced_trees(c, new_row_ids)
if isinstance(node, Leaf):
map_rows_cols_to_node_id[row_ids, node.scope] = node.id
return
_sample_induced_trees(node, np.arange(data.shape[0]))
return map_rows_cols_to_node_id, lls
def run_oSLRAU(dataset, update_after_no_min_batches, prune_after):
data = get_data(dataset)
data = np.where(np.isnan(data),
np.ma.array(data, mask=np.isnan(data)).mean(axis=0), data)
from sklearn.model_selection import train_test_split
train_data, test_data = train_test_split(data,
test_size=0.33,
random_state=42)
# make first mini_batch from data
mini_batch_size = 50
first_mini_batch = data[0:mini_batch_size]
n = first_mini_batch.shape[1] # num of variables
print(n)
context = [Gaussian] * n
ds_context = Context(
parametric_types=context).add_domains(first_mini_batch)
# Learn initial spn
spn = learn_parametric(first_mini_batch, ds_context)
plot_spn(spn, 'intitial_spn.pdf')
print(np.mean(log_likelihood(spn, test_data)))
oSLRAU_params = oSLRAUParams(mergebatch_threshold=128,
corrthresh=0.1,
mvmaxscope=1,
equalweight=True,
currVals=True)
no_of_minibatches = int(data.shape[0] / mini_batch_size)
# update using oSLRAU
for i in range(1, no_of_minibatches):
mini_batch = data[i * mini_batch_size:(i + 1) * mini_batch_size]
update_structure = False
if update_after_no_min_batches // i == 0:
print(i)
update_structure = True
spn = oSLRAU(spn, mini_batch, oSLRAU_params, update_structure)
if i == prune_after:
spn = Prune_oSLRAU(spn)
print(np.mean(log_likelihood(spn, test_data)))
plot_spn(spn, 'final_spn.pdf')
def entropy(spn, ds_context, RVset, debug=False):
"""
calc the entropy from spn and the permutation of RVs
:param spn: input SPN
:param ds_context:
:param RVset: set of scope integers representing RVs
:return: entropy of RVs
"""
# first check if input RVset is type of set
check_set(RVset)
# then check if the RVs are all DISCRETE
check_discrete(ds_context, RVset)
# get permutation of RVset
perm_RV = get_permutation(ds_context, RVset)
# get entropy
from spn.algorithms.Inference import log_likelihood
log_p = log_likelihood(spn, perm_RV)
log_p[np.isinf(log_p)] = 0
h = np.exp(log_p) * log_p
# check, if p==0, log_p will be "-np.inf" and h will be NaN
# if h==NaN, setting it 0 makes entropy=0
h[np.isnan(h)] = 0
H = -(h.sum())
logger.debug("H(%s)=%s" % (RVset, H))
return H
def test_log_vector_histogram():
# Construct a minimal SPN.
h1 = Histogram([0., 1., 2.], [0.25, 0.75], [1, 1], scope=0)
h2 = Histogram([0., 1., 2.], [0.45, 0.55], [1, 1], scope=1)
h3 = Histogram([0., 1., 2.], [0.33, 0.67], [1, 1], scope=0)
h4 = Histogram([0., 1., 2.], [0.875, 0.125], [1, 1], scope=1)
p0 = Product(children=[h1, h2])
p1 = Product(children=[h3, h4])
spn = Sum([0.3, 0.7], [p0, p1])
inputs = np.column_stack((
np.random.randint(2, size=30),
np.random.randint(2, size=30),
)).astype("float64")
if not CPUCompiler.isVectorizationSupported():
print("Test not supported by the compiler installation")
return 0
# Execute the compiled Kernel.
results = CPUCompiler(maxTaskSize=5).log_likelihood(spn, inputs, supportMarginal=False)
# Compute the reference results using the inference from SPFlow.
reference = log_likelihood(spn, inputs)
reference = reference.reshape(30)
# Check the computation results against the reference
# Check in normal space if log-results are not very close to each other.
assert np.all(np.isclose(results, reference)) or np.all(np.isclose(np.exp(results), np.exp(reference)))
def extend():
import numpy as np
from spn.structure.leaves.parametric.Parametric import Leaf
class Pareto(Leaf):
def __init__(self, a, scope=None):
Leaf.__init__(self, scope=scope)
self.a = a
def pareto_likelihood(node, data=None, dtype=np.float64):
probs = np.ones((data.shape[0], 1), dtype=dtype)
from scipy.stats import pareto
probs[:] = pareto.pdf(data[:, node.scope], node.a)
return probs
from spn.algorithms.Inference import add_node_likelihood
add_node_likelihood(Pareto, pareto_likelihood)
spn = 0.3 * Pareto(2.0, scope=0) + 0.7 * Pareto(3.0, scope=0)
from spn.algorithms.Inference import log_likelihood
print("pareto", log_likelihood(spn, np.array([1.5]).reshape(-1, 1)))
def test_bcpp(self):
D = Gaussian(mean=1.0, stdev=1.0, scope=[0])
E = Gaussian(mean=2.0, stdev=2.0, scope=[1])
F = Gaussian(mean=3.0, stdev=3.0, scope=[0])
G = Gaussian(mean=4.0, stdev=4.0, scope=[1])
B = D * E
C = F * G
A = 0.3 * B + 0.7 * C
spn_cc_eval_func = get_cpp_function(A)
np.random.seed(17)
data = np.random.normal(10, 0.01,
size=200000).tolist() + np.random.normal(
30, 10, size=200000).tolist()
data = np.array(data).reshape((-1, 2))
py_ll = log_likelihood(A, data)
c_ll = spn_cc_eval_func(data)
for i in range(py_ll.shape[0]):
self.assertAlmostEqual(py_ll[i, 0], c_ll[i, 0])
def test_cpu_histogram():
# Construct a minimal SPN.
h1 = Histogram([0., 1., 2.], [0.25, 0.75], [1, 1], scope=0)
h2 = Histogram([0., 3., 6., 8.], [0.35, 0.1, 0.55], [1, 1], scope=1)
h3 = Histogram([0., 1., 2.], [0.33, 0.67], [1, 1], scope=0)
h4 = Histogram([0., 5., 8.], [0.875, 0.125], [1, 1], scope=1)
p0 = Product(children=[h1, h2])
p1 = Product(children=[h3, h4])
spn = Sum([0.3, 0.7], [p0, p1])
inputs = np.column_stack((
np.random.randint(2, size=30),
np.random.randint(8, size=30),
)).astype("float64")
# Insert some NaN in random places into the input data.
inputs.ravel()[np.random.choice(inputs.size, 5, replace=False)] = np.nan
if not CUDACompiler.isAvailable():
print("Test not supported by the compiler installation")
return 0
# Execute the compiled Kernel.
results = CUDACompiler().log_likelihood(spn, inputs)
# Compute the reference results using the inference from SPFlow.
reference = log_likelihood(spn, inputs)
reference = reference.reshape(30)
# Check the computation results against the reference
# Check in normal space if log-results are not very close to each other.
assert np.all(np.isclose(results, reference)) or np.all(
np.isclose(np.exp(results), np.exp(reference)))
def test_eval_parametric(self):
data = np.array([1, 1, 1, 1, 1, 1, 1], dtype=np.float32).reshape(
(1, 7))
spn = (Gaussian(mean=1.0, stdev=1.0, scope=[0]) *
Exponential(l=1.0, scope=[1]) *
Gamma(alpha=1.0, beta=1.0, scope=[2]) *
LogNormal(mean=1.0, stdev=1.0, scope=[3]) *
Poisson(mean=1.0, scope=[4]) * Bernoulli(p=0.6, scope=[5]) *
Categorical(p=[0.1, 0.2, 0.7], scope=[6]))
ll = log_likelihood(spn, data)
tf_ll = eval_tf(spn, data)
self.assertTrue(np.all(np.isclose(ll, tf_ll)))
spn_copy = Copy(spn)
tf_graph, data_placeholder, variable_dict = spn_to_tf_graph(
spn_copy, data, 1)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
tf_graph_to_spn(variable_dict)
str_val = spn_to_str_equation(spn)
str_val2 = spn_to_str_equation(spn_copy)
self.assertEqual(str_val, str_val2)
def test_cuda_categorical():
# Construct a minimal SPN
c1 = Categorical(p=[0.35, 0.55, 0.1], scope=0)
c2 = Categorical(p=[0.25, 0.625, 0.125], scope=1)
c3 = Categorical(p=[0.5, 0.2, 0.3], scope=2)
c4 = Categorical(p=[0.6, 0.15, 0.25], scope=3)
c5 = Categorical(p=[0.7, 0.11, 0.19], scope=4)
c6 = Categorical(p=[0.8, 0.14, 0.06], scope=5)
p = Product(children=[c1, c2, c3, c4, c5, c6])
# Randomly sample input values.
inputs = np.column_stack((
np.random.randint(3, size=30),
np.random.randint(3, size=30),
np.random.randint(3, size=30),
np.random.randint(3, size=30),
np.random.randint(3, size=30),
np.random.randint(3, size=30),
)).astype("float64")
if not CUDACompiler.isAvailable():
print("Test not supported by the compiler installation")
return 0
# Execute the compiled Kernel.
results = CUDACompiler().log_likelihood(p, inputs, supportMarginal=False)
# Compute the reference results using the inference from SPFlow.
reference = log_likelihood(p, inputs)
reference = reference.reshape(30)
# Check the computation results against the reference
# Check in normal space if log-results are not very close to each other.
assert np.all(np.isclose(results, reference)) or np.all(
np.isclose(np.exp(results), np.exp(reference)))
def predict(spn):
data = get_data_in_window(features=values)
res = np.ndarray((Y.shape[0] * Y.shape[1], number_of_classes))
for i in range(number_of_classes):
tmp = data.copy()
tmp[:, -int((window_size**2) / 2)] = i
res[:, i] = log_likelihood(spn, tmp)[:, 0]
return np.argmax(res, axis=1).reshape((Y.shape[0], Y.shape[1]))
def test_gaussian_spn_ll(self):
root = 0.3 * (Gaussian(mean=0, stdev=1, scope=0) * Gaussian(
mean=1, stdev=1, scope=1)) + 0.7 * (Gaussian(
mean=2, stdev=1, scope=0) * Gaussian(mean=3, stdev=1, scope=1))
sympyecc = spn_to_sympy(root)
logsympyecc = spn_to_sympy(root, log=True)
sym_l = float(sympyecc.evalf(subs={"x0": 0, "x1": 0}))
sym_ll = float(logsympyecc.evalf(subs={"x0": 0, "x1": 0}))
data = np.array([0, 0], dtype=np.float).reshape(-1, 2)
self.assertTrue(
np.alltrue(np.isclose(np.log(sym_l), log_likelihood(root, data))))
self.assertTrue(
np.alltrue(np.isclose(sym_ll, log_likelihood(root, data))))
def test_PWL_no_variance(self):
data = np.array([1.0, 1.0]).reshape(-1, 1)
ds_context = Context([MetaType.REAL])
ds_context.add_domains(data)
leaf = create_piecewise_leaf(data, ds_context, scope=[0], hist_source="kde")
prob = np.exp(log_likelihood(leaf, data))
self.assertAlmostEqual(float(prob[0]), 2 / 6)
self.assertAlmostEqual(float(prob[1]), 2 / 6)
def run_spflow(spflow_spn, n_feats, batch_size, repetitions):
print("Running SPFlow with: nfeat=%s, batch=%s" % (n_feats, batch_size))
x = np.random.rand(batch_size, n_feats).astype(np.float32)
# warmup
for i in range(10):
ll = log_likelihood(spflow_spn, x)
# Run SPFlow spn
t = 0.0
for i in tqdm(range(repetitions), desc="Repetition loop"):
x = np.random.rand(batch_size, n_feats).astype(np.float32)
t0 = time()
ll = log_likelihood(spflow_spn, x)
t += time() - t0
spflow_time = t / repetitions
return spflow_time
def assert_correct(self, node, x, result):
self.tested.add(type(node))
data = np.array([x], dtype=np.float).reshape(-1, 1)
node.scope = [0]
l = likelihood(node, data)
self.assertAlmostEqual(result, l[0, 0], 5)
self.assertTrue(np.alltrue(np.isclose(np.log(l), log_likelihood(node, data))))
data = np.random.rand(10, 10)
data[:, 5] = x
node.scope = [5]
l = likelihood(node, data)
self.assertEqual(l.shape[0], data.shape[0])
self.assertEqual(l.shape[1], 1)
self.assertTrue(np.isclose(np.var(l), 0))
self.assertTrue(np.alltrue(np.isclose(result, l[0, 0])))
self.assertTrue(np.alltrue(np.isclose(np.log(l), log_likelihood(node, data))))
def test_optimization(self):
np.random.seed(17)
data = np.random.normal(10, 0.01,
size=2000).tolist() + np.random.normal(
30, 10, size=2000).tolist()
data = np.array(data).reshape((-1, 10))
data = data.astype(np.float32)
ds_context = Context(meta_types=[MetaType.REAL] * data.shape[1],
parametric_types=[Gaussian] * data.shape[1])
spn = learn_parametric(data, ds_context)
spn.weights = [0.8, 0.2]
py_ll = log_likelihood(spn, data)
tf_graph, data_placeholder, variable_dict = spn_to_tf_graph(spn, data)
loss = likelihood_loss(tf_graph)
output = tf.train.AdamOptimizer(0.001).minimize(loss)
with tf.Session() as session:
session.run(tf.global_variables_initializer())
for step in range(50):
session.run(output, feed_dict={data_placeholder: data})
# print("loss:", step, session.run(-loss, feed_dict={data_placeholder: data}))
tf_ll_opt = session.run(tf_graph,
feed_dict={
data_placeholder: data
}).reshape(-1, 1)
tf_graph_to_spn(variable_dict)
py_ll_opt = log_likelihood(spn, data)
# print(tf_ll_opt.sum(), py_ll_opt.sum())
self.assertTrue(np.all(np.isclose(tf_ll_opt, py_ll_opt)))
self.assertLess(py_ll.sum(), tf_ll_opt.sum())
def inference():
import numpy as np
spn = create_SPN()
spn_marg = marginalize()
test_data = np.array([1.0, 0.0, 1.0]).reshape(-1, 3)
from spn.algorithms.Inference import log_likelihood
ll = log_likelihood(spn, test_data)
print("python ll", ll, np.exp(ll))
llm = log_likelihood(spn_marg, test_data)
print("python ll spn_marg", llm, np.exp(llm))
test_data2 = np.array([np.nan, 0.0, 1.0]).reshape(-1, 3)
llom = log_likelihood(spn, test_data2)
print("python ll spn with nan", llom, np.exp(llom))
def apply(train_datasets, ds_contexts, test_datasets, n_folds, result_path, filename, foldLog):
# Comment this if you are interested in seen the warnings, we observed that many informative warnings are
# thrown here, but didn't see nothing suspicious, simly executing Spflow's Mspn method
warnings.filterwarnings("ignore", category=RuntimeWarning)
warnings.filterwarnings("ignore", category=UserWarning)
print("\n========================")
print("MSPN")
print("========================")
results = {}
folds = {}
avg_learning_time = 0
avg_test_ll = 0
for i in range(1, n_folds + 1):
index = i-1
# Only for MSPN:
ds_contexts[index].add_domains(train_datasets[index])
init_time = time.time()*1000
model = learn_mspn(train_datasets[index], ds_contexts[index], min_instances_slice=20)
end_time = time.time()*1000
learning_time = end_time - init_time
test_ll = log_likelihood(model, test_datasets[index])
test_ll = np.sum(test_ll)
fold_result = {"test_ll": test_ll, "learning_time": learning_time}
folds["fold_" + str(i)] = fold_result
avg_learning_time = avg_learning_time + learning_time
avg_test_ll = avg_test_ll + test_ll
if foldLog:
print("----------------------------------------")
print("Fold (" + str(i) + "): ")
print("Test LL: " + str(test_ll))
print("Learning time: " + str(learning_time))
# Generate the average results and store them in the dictionary, then store them in a JSON file
avg_test_ll = avg_test_ll / n_folds
avg_learning_time = avg_learning_time / n_folds / 1000 # in seconds
results["average_test_ll"] = avg_test_ll
results["average_learning_time"] = avg_learning_time
results["folds"] = folds
store_json(results, result_path, filename)
print("----------------------------------------")
print("----------------------------------------")
print("Average Test LL: " + str(avg_test_ll))
print("Average learning time: " + str(avg_learning_time))
def assert_correct(self, node, x, result):
self.tested.add(type(node))
data = np.array([x], dtype=np.float).reshape(1, -1)
node.scope = list(range(data.shape[1]))
l = likelihood(node, data)
self.assertAlmostEqual(result, l[0, 0], 5)
self.assertTrue(
np.alltrue(np.isclose(np.log(l), log_likelihood(node, data))))
new_scope = (np.array(node.scope) + 5).tolist()
data = np.random.rand(10, max(new_scope) + 2)
data[:, new_scope] = x
node.scope = new_scope
l = likelihood(node, data)
self.assertEqual(l.shape[0], data.shape[0])
self.assertEqual(l.shape[1], 1)
self.assertTrue(np.isclose(np.var(l), 0))
self.assertTrue(np.alltrue(np.isclose(result, l[0, 0])))
self.assertTrue(
np.alltrue(np.isclose(np.log(l), log_likelihood(node, data))))
