def get_node_description(spn, parent_node, size):
# parent_node.validate()
parent_type = type(parent_node).__name__
node_descriptions = dict()
node_descriptions['num'] = len(parent_node.children)
nodes = list()
for i, node in enumerate(parent_node.children):
node_spn = Copy(node)
assign_ids(node_spn)
node_dir = dict()
node_dir[
'weight'] = parent_node.weights[i] if parent_type == 'Sum' else 1
node_dir['size'] = get_number_of_nodes(node) - 1
node_dir['num_children'] = len(
node.children) if not isinstance(node, Leaf) else 0
node_dir['leaf'] = isinstance(node, Leaf)
node_dir['type'] = type(node).__name__ + ' Node'
node_dir['split_features'] = [
list(c.scope) for c in node.children
] if not isinstance(node, Leaf) else node.scope
node_dir['split_features'].sort(key=lambda x: len(x))
node_dir['depth'] = get_depth(node)
node_dir['child_depths'] = [get_depth(c) for c in node.children]
descriptor = node_dir['type']
if all((d == 0 for d in node_dir['child_depths'])):
descriptor = 'shallow ' + descriptor
node_dir['quick'] = 'shallow'
elif len([d for d in node_dir['child_depths'] if d == 0]) == 1:
node_dir['quick'] = 'split_one'
descriptor += ', which separates one feature'
else:
node_dir['quick'] = 'deep'
descriptor = 'deep ' + descriptor
descriptor = 'a ' + descriptor
node_dir['descriptor'] = descriptor
node_dir['short_descriptor'] = descriptor
node_dir['representative'] = mpe(node_spn, np.array([[np.nan] * size]))
nodes.append(node_dir)
node_descriptions['shallow'] = len(
[d for d in nodes if d['quick'] == 'shallow'])
node_descriptions['split_one'] = len(
[d for d in nodes if d['quick'] == 'split_one'])
node_descriptions['deep'] = len([d for d in nodes if d['quick'] == 'deep'])
nodes.sort(key=lambda x: x['weight'])
nodes.reverse()
node_descriptions['nodes'] = nodes
return node_descriptions
def get_structure_stats(node):
num_nodes = len(get_nodes_by_type(node, Node))
sum_nodes = get_nodes_by_type(node, Sum)
n_sum_nodes = len(sum_nodes)
n_prod_nodes = len(get_nodes_by_type(node, Product))
leaf_nodes = get_nodes_by_type(node, Leaf)
n_leaf_nodes = len(leaf_nodes)
edges = get_number_of_edges(node)
layers = get_depth(node)
params = 0
for n in sum_nodes:
params += len(n.children)
for l in leaf_nodes:
params += len(l.parameters)
return """---Structure Statistics---
# nodes %s
# sum nodes %s
# prod nodes %s
# leaf nodes %s
# params %s
# edges %s
# layers %s""" % (
num_nodes,
n_sum_nodes,
n_prod_nodes,
n_leaf_nodes,
params,
edges,
layers,
)
def get_structure_stats_dict(node):
node_types = dict(Counter([type(n) for n in get_nodes_by_type(node)]))
num_nodes = len(get_nodes_by_type(node, Node))
edges = get_number_of_edges(node)
layers = get_depth(node)
return {
'nodes': num_nodes,
'edges': edges,
'layers': layers
}.update(node_types)
def run_experiment(exp, spn, test_data, test_type, exp_lambda):
outprefix = path + "/spns/%s/" % (exp)
results_file = "%stime_test_%s_ll_%s.txt" % (outprefix, test_type, OS_name)
if os.path.isfile(results_file):
return
print(exp, test_data.shape, test_type)
ll, test_time = exp_lambda()
np.savetxt(results_file, ll, delimiter=";")
import cpuinfo
machine = cpuinfo.get_cpu_info()["brand"]
adds, muls = fpga_count_ops(spn)
test_n = test_data.shape[0]
results = OrderedDict()
results["Experiment"] = exp
results["OS"] = OS_name
results["machine"] = machine
results["test type"] = test_type
results["expected adds"] = adds
results["expected muls"] = muls
results["input rows"] = test_n
results["input cols"] = test_data.shape[1]
results["spn nodes"] = len(get_nodes_by_type(spn, Node))
results["spn sum nodes"] = len(get_nodes_by_type(spn, Sum))
results["spn prod nodes"] = len(get_nodes_by_type(spn, Product))
results["spn leaves"] = len(get_nodes_by_type(spn, Leaf))
results["spn edges"] = get_number_of_edges(spn)
results["spn layers"] = get_depth(spn)
results["time per task"] = test_time
results["time per instance"] = test_time / test_n
results["avg ll"] = np.mean(ll, dtype=np.float128)
results_file_name = "results.csv"
if not os.path.isfile(results_file_name):
results_file = open(results_file_name, "w")
results_file.write(";".join(results.keys()))
results_file.write("\n")
else:
results_file = open(results_file_name, "a")
results_file.write(";".join(map(str, results.values())))
results_file.write("\n")
results_file.close()
def get_structure_stats_dict(node):
node_types = dict(Counter([type(n) for n in get_nodes_by_type(node)]))
num_nodes = len(get_nodes_by_type(node, Node))
edges = get_number_of_edges(node)
layers = get_depth(node)
result = {
"nodes": num_nodes,
"edges": edges,
"layers": layers,
"count_per_type": node_types
}
return result
def get_structure_stats(node):
num_nodes = len(get_nodes_by_type(node, Node))
sum_nodes = len(get_nodes_by_type(node, Sum))
prod_nodes = len(get_nodes_by_type(node, Product))
leaf_nodes = len(get_nodes_by_type(node, Leaf))
edges = get_number_of_edges(node)
layers = get_depth(node)
return """---Structure Statistics---
# nodes %s
# sum nodes %s
# prod nodes %s
# leaf nodes %s
# edges %s
# layers %s""" % (num_nodes, sum_nodes, prod_nodes, leaf_nodes,
edges, layers)
def complete_layers(layer_nodes, current_node_type=Sum, depth=None):
# all leaves should be at same depth
root_layer = False
if depth is None:
root_layer = True
depth = get_depth(layer_nodes[0])
if depth == 2:
return
children_layer = []
if current_node_type == Sum:
for i in range(len(layer_nodes)):
n = layer_nodes[i]
assert isinstance(n, Sum)
for j in range(len(n.children)):
c = n.children[j]
if not isinstance(c, Product):
n.children[j] = Product([c])
children_layer.extend(n.children)
children_layer_type = Product
elif current_node_type == Product:
for i in range(len(layer_nodes)):
n = layer_nodes[i]
assert isinstance(n, Product)
for j in range(len(n.children)):
c = n.children[j]
if not isinstance(c, Sum):
n.children[j] = Sum([1.0], [c])
children_layer.extend(n.children)
children_layer_type = Sum
else:
raise Exception('node type' + str(current_node_type))
complete_layers(children_layer,
current_node_type=children_layer_type,
depth=depth - 1)
if root_layer:
rebuild_scopes_bottom_up(layer_nodes[0])
assign_ids(layer_nodes[0])
def node_correlation(spn, dictionary):
all_nodes = list(i for i, node in enumerate(spn.children)
if get_depth(node) > 1)
if nodes == 'all':
shown_nodes = all_nodes
elif isinstance(nodes, int):
num_choices = min(nodes, len(all_nodes))
shown_nodes = random.sample(all_nodes, k=num_choices)
else:
shown_nodes = nodes
shown_nodes = [spn.children[i] for i in shown_nodes]
node_descritions = get_node_description(spn, spn, len(spn.scope))
used_descriptions = [
node_descritions['nodes'][i] for i, _ in enumerate(shown_nodes)
]
for i, (node, d) in enumerate(zip(shown_nodes, used_descriptions)):
if not d['quick'] == 'shallow':
printmd('### Correlations for node {}'.format(i))
correlation_description(node, dictionary)
def get_structure_stats_dict(node):
nodes = get_nodes_by_type(node, Node)
num_nodes = len(nodes)
node_types = dict(Counter([type(n) for n in nodes]))
edges = get_number_of_edges(node)
layers = get_depth(node)
params = 0
for n in nodes:
if isinstance(n, Sum):
params += len(n.children)
if isinstance(n, Leaf):
params += len(n.parameters)
result = {
"nodes": num_nodes,
"params": params,
"edges": edges,
"layers": layers,
"count_per_type": node_types
}
return result